v7.1.0: Obscureable URL query parameters (JaffaKetchup#95)

* Added URL query params obscurer feature Added `headers` and `httpClient` parameters to `getTileProvider` Minor documentation improvements Minor bug fixes * Built Example Applications --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

Author: JaffaKetchup

CHANGELOG.md

@@ -13,6 +13,13 @@ Many thanks to my sponsors, no matter how much or how little they donated. Spons
 
 # Changelog
 
+## [7.1.0] - 2023/02/12
+
+* Added URL query params obscurer feature
+* Added `headers` and `httpClient` parameters to `getTileProvider`
+* Minor documentation improvements
+* Minor bug fixes
+
 ## [7.0.2] - 2023/02/12
 
 * Minor changes to example application

lib/src/providers/image_provider.dart

@@ -2,7 +2,6 @@
 // A full license can be found at .\LICENSE
 
 import 'dart:async';
-import 'dart:io';
 import 'dart:ui';
 
 import 'package:flutter/foundation.dart';
@@ -29,12 +28,6 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
  /// The coordinates of the tile to be fetched
  final Coords<num> coords;
 
- /// A HTTP client used to send requests
- final HttpClient httpClient;
-
- /// Custom headers to send with each request
- final Map<String, String> headers;
-
  /// Used internally to safely and efficiently enforce the `settings.maxStoreLength`
  static final Queue removeOldestQueue =
  Queue(timeout: const Duration(seconds: 1));
@@ -55,8 +48,6 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
  required this.provider,
  required this.options,
  required this.coords,
- required this.httpClient,
- required this.headers,
  }) : settings = provider.settings,
  _db = FMTCRegistry.instance(provider.storeDirectory.storeName);
 
@@ -112,11 +103,8 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
  try {
  throw FMTCBrowsingError(throwError, throwErrorType!);
  } on FMTCBrowsingError catch (e) {
- if (settings.errorHandler != null) {
- settings.errorHandler!(e);
- } else {
- rethrow;
- }
+ settings.errorHandler?.call(e);
+ rethrow;
  }
  }
 
@@ -125,12 +113,27 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
  }
 
  throw ArgumentError(
- '`finish` was called with an invalid combination of arguments',
+ '`finish` was called with an invalid combination of arguments, or a fall-through situation occurred.',
  );
  }
 
- final String url = provider.getTileUrl(coords, options);
- final DbTile? existingTile = await _db.tiles.get(DatabaseTools.hash(url));
+ final String networkUrl = provider.getTileUrl(coords, options);
+ final String matcherUrl;
+
+ if (networkUrl.contains('?') &&
+ provider.settings.obscuredQueryParams.isNotEmpty) {
+ String secondPartUrl = networkUrl.split('?')[1];
+ for (final r in provider.settings.obscuredQueryParams) {
+ secondPartUrl = secondPartUrl.replaceAll(r, '');
+ }
+
+ matcherUrl = '${networkUrl.split('?')[0]}?$secondPartUrl';
+ } else {
+ matcherUrl = networkUrl;
+ }
+
+ final DbTile? existingTile =
+ await _db.tiles.get(DatabaseTools.hash(matcherUrl));
 
  // Logic to check whether the tile needs creating or updating
  final bool needsCreating = existingTile == null;
@@ -142,6 +145,15 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
  existingTile.lastModified.millisecondsSinceEpoch >
  settings.cachedValidDuration.inMilliseconds);
 
+ /* DEBUG ONLY
+ print('---------');
+ print(networkUrl);
+ print(matcherUrl);
+ print(' Existing ID: ' + (existingTile?.id ?? 'None').toString());
+ print(' Needs Creating: ' + needsCreating.toString());
+ print(' Needs Updating: ' + needsUpdating.toString());
+ */
+
  // Get any existing bytes from the tile, if it exists
  Uint8List? bytes;
  if (!needsCreating) bytes = Uint8List.fromList(existingTile.bytes);
@@ -162,9 +174,10 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
 
  // Try to get a response from a server, throwing an error if not possible & the tile does not exist
  try {
- final HttpClientRequest request =
- await httpClient.getUrl(Uri.parse(url));
- headers.forEach((k, v) => request.headers.add(k, v));
+ final request = await provider.httpClient.getUrl(Uri.parse(networkUrl));
+ provider.headers.forEach(
+ (k, v) => request.headers.add(k, v, preserveHeaderCase: true),
+ );
  response = await request.close();
  } catch (err) {
  return finish(
@@ -204,7 +217,9 @@ class FMTCImageProvider extends ImageProvider<FMTCImageProvider> {
 
  // Cache the tile asynchronously
  unawaited(
- _db.writeTxn(() => _db.tiles.put(DbTile(url: url, bytes: bytes!))),
+ _db.writeTxn(
+ () => _db.tiles.put(DbTile(url: matcherUrl, bytes: bytes!)),
+ ),
  );
 
  // If an new tile was created over the tile limit, delete the oldest tile

lib/src/providers/tile_provider.dart

@@ -3,55 +3,56 @@
 
 part of flutter_map_tile_caching;
 
-/// 'flutter_map_tile_caching's custom [TileProvider] for use in a [TileLayer]
+/// FMTC's custom [TileProvider] for use in a [TileLayer]
+///
+/// Create from the store directory chain, eg. [StoreDirectory.getTileProvider].
 class FMTCTileProvider extends TileProvider {
  /// The store directory attached to this provider
  final StoreDirectory storeDirectory;
 
  /// The tile provider settings to use
  ///
- /// Defaults to the one provided by [FMTCSettings] when initialising [FlutterMapTileCaching].
+ /// Defaults to the one provided by [FMTCSettings] when initialising
+ /// [FlutterMapTileCaching].
  final FMTCTileProviderSettings settings;
 
  /// Used internally for browsing-caused tile requests
  final HttpClient httpClient;
 
- /// 'flutter_map_tile_caching's custom [TileProvider] for use in a [TileLayer]
- ///
- /// Usually created from the store directory chain, eg. [StoreDirectory.getTileProvider].
- ///
- /// This contains the logic for the tile provider, such as browse caching and using bulk downloaded tiles.
- FMTCTileProvider({
+ FMTCTileProvider._({
  required this.storeDirectory,
  required FMTCTileProviderSettings? settings,
- super.headers,
+ Map<String, String> headers = const {},
  HttpClient? httpClient,
  }) : settings =
  settings ?? FMTC.instance.settings.defaultTileProviderSettings,
  httpClient = httpClient ?? HttpClient()
- ..userAgent = null;
+ ..userAgent = null,
+ super(
+ headers: {
+ ...headers,
+ 'User-Agent': headers['User-Agent'] == null
+ ? 'flutter_map_tile_caching for flutter_map (unknown)'
+ : 'flutter_map_tile_caching for ${headers['User-Agent']}',
+ },
+ );
 
- /// Closes the open [HttpClient] - this will make the provider unable to perform network requests
+ /// Closes the open [HttpClient] - this will make the provider unable to
+ /// perform network requests
  @override
  void dispose() {
- super.dispose();
  httpClient.close();
+ super.dispose();
  }
 
- /// Get a browsed tile as an image, paint it on the map and save it's bytes to cache for later (dependent on the [CacheBehavior])
+ /// Get a browsed tile as an image, paint it on the map and save it's bytes to
+ /// cache for later (dependent on the [CacheBehavior])
  @override
  ImageProvider getImage(Coords<num> coords, TileLayer options) =>
  FMTCImageProvider(
  provider: this,
  options: options,
  coords: coords,
- httpClient: httpClient,
- headers: {
- ...headers,
- 'User-Agent': headers['User-Agent'] == null
- ? 'flutter_map_tile_caching for flutter_map (unknown)'
- : 'flutter_map_tile_caching for ${headers['User-Agent']}',
- },
  );
 
  IsarCollection<DbTile> get _tiles =>
@@ -64,7 +65,8 @@ class FMTCTileProvider extends TileProvider {
  }) =>
  _tiles.getSync(DatabaseTools.hash(getTileUrl(coords, options))) != null;
 
- /// Check whether a specified tile is cached in the current store asynchronously
+ /// Check whether a specified tile is cached in the current store
+ /// asynchronously
  Future<bool> checkTileCachedAsync({
  required Coords<num> coords,
  required TileLayer options,

lib/src/settings/fmtc_settings.dart

@@ -3,23 +3,24 @@
 
 part of flutter_map_tile_caching;
 
-/// Global FMTC settings, used throughout or in long lasting places
+/// Global FMTC settings
 class FMTCSettings {
  /// Default settings used when creating an [FMTCTileProvider]
  ///
- /// Can be overridden on a case-to-case basis when actually creating the tile provider.
+ /// Can be overridden on a case-to-case basis when actually creating the tile
+ /// provider.
  final FMTCTileProviderSettings defaultTileProviderSettings;
 
  /// Sets a strict upper size limit on each underlying database individually
  /// (of which there are multiple)
  ///
- /// Prefer to set a limit on the number of tiles instead, using
- /// [FMTCTileProviderSettings.maxStoreLength].
+ /// It is also recommended to set a limit on the number of tiles instead, using
+ /// [FMTCTileProviderSettings.maxStoreLength]. If using a generous number
+ /// there, use a larger number here as well.
  ///
- /// Setting this value too low may cause errors. Setting this value too high
- /// and not limiting the number of tiles may result in slower operations and
- /// a negative user experience: a large, unknown file may be deleted by a user,
- /// causing significant data loss.
+ /// Setting this value too low may cause unexpected errors when writing to the
+ /// database. Setting this value too high may cause memory issues on certain
+ /// older devices or emulators.
  ///
  /// Defaults to 2GiB (2048MiB).
  final int databaseMaxSize;
@@ -39,8 +40,7 @@ class FMTCSettings {
  /// exporting a store will always compact it's underlying database.
  final DatabaseCompactCondition? databaseCompactCondition;
 
- /// Create custom global FMTC settings, used throughout or in long lasting
- /// places
+ /// Create custom global FMTC settings
  FMTCSettings({
  FMTCTileProviderSettings? defaultTileProviderSettings,
  this.databaseMaxSize = 2048,

lib/src/settings/tile_provider_settings.dart

@@ -1,24 +1,25 @@
 // Copyright © Luka S (JaffaKetchup) under GPL-v3
 // A full license can be found at .\LICENSE
 
-import 'package:isar/isar.dart';
-
 import '../../flutter_map_tile_caching.dart';
 import '../providers/image_provider.dart';
 
-/// Multiple behaviors dictating how browse caching should be carried out
-///
-/// Check documentation on each value for more information.
+/// Behaviours dictating how and when browse caching should be carried out
 enum CacheBehavior {
  /// Only get tiles from the local cache
  ///
- /// Useful for applications with dedicated 'Offline Mode'.
+ /// Throws [FMTCBrowsingErrorType.missingInCacheOnlyMode] if a tile is not
+ /// available.
  cacheOnly,
 
- /// Get tiles from the local cache, going on the Internet to update the cached tile if it has expired (`cachedValidDuration` has passed)
+ /// Get tiles from the local cache, only using the network to update the cached
+ /// tile if it has expired ([FMTCTileProviderSettings.cachedValidDuration] has
+ /// passed)
  cacheFirst,
 
- /// Get tiles from the Internet and update the cache for every tile
+ /// Get tiles from the network where possible, and update the cached tiles
+ ///
+ /// Safely falls back to using cached tiles if the network is not available.
  onlineFirst,
 }
 
@@ -44,26 +45,31 @@ class FMTCTileProviderSettings {
  /// Only applies to 'browse caching', ie. downloading regions will bypass this
  /// limit.
  ///
- /// Note that the actual store has an un-modifiable maximum size limit of
- /// [Isar.defaultMaxSizeMiB] (1GB). It is unspecified what will happen if this
- /// limit is reached, however it is likely that an error will be thrown.
+ /// Note that the actual store has a size limit of
+ /// [FMTCSettings.databaseMaxSize], irrespective of this value.
  ///
  /// Defaults to 0 disabled.
  final int maxStoreLength;
 
+ /// A list of keys in the query part of the source URL (after the '?'), who's
+ /// values will not be stored
+ ///
+ /// See the online documentation for more information.
+ final Iterable<RegExp> obscuredQueryParams;
+
  /// A custom callback that will be called when an [FMTCBrowsingError] is raised
  ///
- /// Prevents the error being printed to the console, and only captures this
- /// type of error, unlike 'flutter_map's native solution.
+ /// Even if this is defined, the error will still be (re)thrown.
  void Function(FMTCBrowsingError exception)? errorHandler;
 
  /// Create settings for an [FMTCTileProvider]
  FMTCTileProviderSettings({
  this.behavior = CacheBehavior.cacheFirst,
  this.cachedValidDuration = const Duration(days: 16),
  this.maxStoreLength = 0,
+ List<String> obscuredQueryParams = const [],
  this.errorHandler,
- });
+ }) : obscuredQueryParams = obscuredQueryParams.map((e) => RegExp('$e=[^&]*'));
 
  @override
  bool operator ==(Object other) =>

lib/src/store/directory.dart

@@ -48,14 +48,23 @@ class StoreDirectory {
  /// background downloading functionality.
  DownloadManagement get download => DownloadManagement._(this);
 
- /// Get 'flutter_map_tile_caching's custom [TileProvider] for use in a
- /// [TileLayer], specific to this store
+ /// Get the [TileProvider] suitable to connect the [TileLayer] to FMTC's
+ /// internals
  ///
  /// Uses [FMTCSettings.defaultTileProviderSettings] by default (and it's
  /// default if unspecified). Alternatively, override [settings] for this get
  /// only.
- FMTCTileProvider getTileProvider([FMTCTileProviderSettings? settings]) =>
- FMTCTileProvider(storeDirectory: this, settings: settings);
+ FMTCTileProvider getTileProvider([
+ FMTCTileProviderSettings? settings,
+ Map<String, String>? headers,
+ HttpClient? httpClient,
+ ]) =>
+ FMTCTileProvider._(
+ storeDirectory: this,
+ settings: settings,
+ headers: headers ?? {},
+ httpClient: httpClient,
+ );
 
  @override
  bool operator ==(Object other) =>

prebuiltExampleApplications/AndroidApplication.apk

@@ -1,7 +1,7 @@
 name: flutter_map_tile_caching
 description: Plugin for 'flutter_map' providing advanced caching functionality,
  with ability to download map regions for offline use.
-version: 7.0.2
+version: 7.1.0
 repository: https://github.com/JaffaKetchup/flutter_map_tile_caching
 issue_tracker: https://github.com/JaffaKetchup/flutter_map_tile_caching/issues
 documentation: https://fmtc.jaffaketchup.dev