latestPreview* fields in model and updates in backend (#4425)

* latestPreview* fields in model and updates in backend

* Prevent NPE

* Fix NPE in test

* Remove versions.sort()

Author: isoos

app/bin/tools/backfill_package_fields.dart

@@ -9,6 +9,7 @@ import 'package:pool/pool.dart';
 import 'package:pub_dev/package/models.dart';
 import 'package:pub_dev/service/entrypoint/tools.dart';
 import 'package:pub_dev/shared/datastore.dart';
+import 'package:pub_dev/tool/utils/dart_sdk_version.dart';
 
 final _argParser = ArgParser()
   ..addOption('concurrency',
@@ -26,6 +27,7 @@ Future main(List<String> args) async {
     print('Ensures Package.assignedTags is a list.');
     print('Ensures Package.latestPublished is a DateTime.');
     print('Ensures Package.latestPrereleasePublished is a DateTime.');
+    print('Ensures Package.lastVersionPublished is a DateTime.');
     print(_argParser.usage);
     return;
   }
@@ -54,10 +56,12 @@ Future<void> _backfillPackageFields(Package p) async {
       p.isWithheld != null &&
       p.assignedTags != null &&
       p.latestPublished != null &&
-      p.latestPrereleasePublished != null) {
+      p.latestPrereleasePublished != null &&
+      p.lastVersionPublished != null) {
     return;
   }
   print('Backfilling properties on package ${p.name}');
+  final currentDartSdk = await getDartSdkVersion();
   final latestVersion =
       await dbService.lookupValue<PackageVersion>(p.latestVersionKey);
   final latestPrereleaseVersion =
@@ -73,8 +77,13 @@ Future<void> _backfillPackageFields(Package p) async {
       package.isUnlisted ??= false;
       package.isWithheld ??= false;
       package.assignedTags ??= [];
-      package.latestPublished = latestVersion.created;
-      package.latestPrereleasePublished = latestPrereleaseVersion.created;
+      package.latestPublished ??= latestVersion.created;
+      package.latestPrereleasePublished ??= latestPrereleaseVersion.created;
+      // updates lastVersionPublished
+      package.updateVersion(latestVersion,
+          dartSdkVersion: currentDartSdk.semanticVersion);
+      package.updateVersion(latestPrereleaseVersion,
+          dartSdkVersion: currentDartSdk.semanticVersion);
       tx.insert(package);
     });
     print('Updated properties on package ${p.name}.');

app/lib/admin/backend.dart

@@ -25,6 +25,7 @@ import '../shared/configuration.dart';
 import '../shared/datastore.dart';
 import '../shared/exceptions.dart';
 import '../shared/tags.dart';
+import '../tool/utils/dart_sdk_version.dart';
 
 final _logger = Logger('pub.admin.backend');
 final _continuationCodec = utf8.fuse(hex);
@@ -381,6 +382,7 @@ class AdminBackend {
     _logger.info('${caller.userId} (${caller.email}) initiated the delete '
         'of package $packageName $version');
 
+    final currentDartSdk = await getDartSdkVersion();
     await withRetryTransaction(_db, (tx) async {
       final Key packageKey = _db.emptyKey.append(Package, id: packageName);
       final package = (await tx.lookup([packageKey])).first as Package;
@@ -405,16 +407,22 @@ class AdminBackend {
       bool updatePackage = false;
       if (package != null && package.latestVersion == version) {
         package.latestVersionKey = null;
+        package.latestPublished = null;
         updatePackage = true;
       }
       if (package != null && package.latestPrereleaseVersion == version) {
         package.latestPrereleaseVersionKey = null;
+        package.latestPrereleasePublished = null;
         updatePackage = true;
       }
+      if (package != null && package.latestPreviewVersion == version) {
+        package.latestPreviewVersionKey = null;
+        package.latestPreviewPublished = null;
+      }
       if (updatePackage) {
-        versions
-            .where((v) => v.version != version)
-            .forEach(package.updateVersion);
+        package.lastVersionPublished = null;
+        versions.where((v) => v.version != version).forEach((v) => package
+            .updateVersion(v, dartSdkVersion: currentDartSdk.semanticVersion));
         package.updated = DateTime.now().toUtc();
         tx.insert(package);
       }

app/lib/package/backend.dart

@@ -15,8 +15,8 @@ import 'package:logging/logging.dart';
 import 'package:meta/meta.dart';
 import 'package:pana/pana.dart' show runProc;
 import 'package:path/path.dart' as p;
-import 'package:pub_dev/package/overrides.dart';
 import 'package:pub_package_reader/pub_package_reader.dart';
+import 'package:pub_semver/pub_semver.dart';
 
 import '../account/backend.dart';
 import '../account/consent_backend.dart';
@@ -38,9 +38,11 @@ import '../shared/redis_cache.dart' show cache;
 import '../shared/storage.dart';
 import '../shared/urls.dart' as urls;
 import '../shared/utils.dart';
+import '../tool/utils/dart_sdk_version.dart';
 import 'model_properties.dart';
 import 'models.dart';
 import 'name_tracker.dart';
+import 'overrides.dart';
 import 'upload_signer_service.dart';
 
 // The maximum stored length of `README.md` and other user-provided file content
@@ -249,6 +251,79 @@ class PackageBackend {
     return _storage.downloadUrl(package, version);
   }
 
+  /// Updates the stable, prerelease and preview versions of [package].
+  ///
+  /// Returns true if the values did change.
+  Future<bool> updatePackageVersions(
+    String package, {
+    Version dartSdkVersion,
+  }) async {
+    final pkgKey = db.emptyKey.append(Package, id: package);
+    dartSdkVersion ??= (await getDartSdkVersion()).semanticVersion;
+
+    // ordered version list by publish date
+    final versions =
+        await db.query<PackageVersion>(ancestorKey: pkgKey).run().toList();
+
+    return await withRetryTransaction(db, (tx) async {
+      final p = await tx.lookupOrNull<Package>(pkgKey);
+      if (p == null) {
+        throw NotFoundException.resource('package "$package"');
+      }
+
+      final oldStableVersion = p.latestSemanticVersion;
+      final oldPrereleaseVersion = p.latestPrereleaseSemanticVersion;
+      final oldPreviewVersion = p.latestPreviewSemanticVersion;
+
+      // reset field values
+      p.latestVersionKey = null;
+      p.latestPrereleaseVersionKey = null;
+      p.latestPreviewVersionKey = null;
+
+      // update fields
+      for (final pv in versions) {
+        p.updateVersion(pv, dartSdkVersion: dartSdkVersion);
+      }
+
+      // shortcut if there was no change
+      final unchanged = oldStableVersion == p.latestSemanticVersion &&
+          oldPrereleaseVersion == p.latestPrereleaseSemanticVersion &&
+          oldPreviewVersion == p.latestPreviewSemanticVersion;
+      if (unchanged) {
+        return false;
+      }
+
+      // sanity check changes
+      if (oldStableVersion.compareTo(p.latestSemanticVersion) > 0 ||
+          oldPrereleaseVersion.compareTo(p.latestPrereleaseSemanticVersion) >
+              0 ||
+          oldPreviewVersion.compareTo(p.latestPreviewSemanticVersion) > 0) {
+        _logger.severe(
+            'Version update sanity check failed for package "$package": '
+            '$oldStableVersion -> ${p.latestVersion} / '
+            '$oldPrereleaseVersion -> ${p.latestPrereleaseVersion} / '
+            '$oldPreviewVersion -> ${p.latestPreviewVersion}');
+        return false;
+      }
+
+      tx.insert(p);
+      return true;
+    });
+  }
+
+  /// Updates the stable, prerelase and preview versions of all package.
+  ///
+  /// Return the number of updated packages.
+  Future<int> updateAllPackageVersions({Version dartSdkVersion}) async {
+    var count = 0;
+    await for (final p in db.query<Package>().run()) {
+      final updated =
+          await updatePackageVersions(p.name, dartSdkVersion: dartSdkVersion);
+      if (updated) count++;
+    }
+    return count;
+  }
+
   /// Updates [options] on [package].
   Future<void> updateOptions(String package, api.PkgOptions options) async {
     final user = await requireAuthenticatedUser();
@@ -596,6 +671,8 @@ class PackageBackend {
           newVersion.package, _maxVersionsPerPackage);
     }
 
+    final currentDartSdk = await getDartSdkVersion();
+
     Package package;
     String prevLatestStableVersion;
     String prevLatestPrereleaseVersion;
@@ -648,7 +725,8 @@ class PackageBackend {
       newVersion.publisherId = package.publisherId;
 
       // Keep the latest version in the package object up-to-date.
-      package.updateVersion(newVersion);
+      package.updateVersion(newVersion,
+          dartSdkVersion: currentDartSdk.semanticVersion);
       package.updated = DateTime.now().toUtc();
 
       _logger.info(

app/lib/package/model_properties.dart

@@ -39,6 +39,9 @@ class Pubspec {
       pubspek.Pubspec.parse(yamlString, lenient: true),
       json.encode(_loadYaml(yamlString)));
 
+  factory Pubspec.fromJson(Map<String, dynamic> map) =>
+      Pubspec._(pubspek.Pubspec.fromJson(map, lenient: true), json.encode(map));
+
   Map<String, dynamic> get asJson {
     _load();
     return _json;

app/lib/package/models.dart

@@ -60,6 +60,11 @@ class Package extends db.ExpandoModel<String> {
   @db.IntProperty(required: true)
   int likes;
 
+  /// [DateTime] when the most recently uploaded [PackageVersion] was published.
+  /// TODO: set required: true after backfill was done.
+  @db.DateTimeProperty(required: false)
+  DateTime lastVersionPublished;
+
   /// Key referencing the [PackageVersion] for the latest version of this package, ordered by priority order of
   /// semantic versioning, hence, deprioritizing prereleases.
   @db.ModelKeyProperty(propertyName: 'latest_version', required: true)
@@ -81,6 +86,18 @@ class Package extends db.ExpandoModel<String> {
   @db.DateTimeProperty()
   DateTime latestPrereleasePublished;
 
+  /// Reference to latest version of this package ordered by semantic versioning,
+  /// filtered for versions that depend on an SDK that will be published in the
+  /// future.
+  ///
+  /// Note: the version may be stable or prerelease.
+  @db.ModelKeyProperty()
+  db.Key latestPreviewVersionKey;
+
+  /// DateTime at which point the `PackageVersion` referenced in [latestPreviewVersionKey] was published.
+  @db.DateTimeProperty()
+  DateTime latestPreviewPublished;
+
   /// The publisher id (null, if the package does not have a publisher).
   @db.StringProperty()
   String publisherId;
@@ -144,14 +161,27 @@ class Package extends db.ExpandoModel<String> {
   String get latestPrereleaseVersion =>
       latestPrereleaseVersionKey?.id as String;
 
+  String get latestPreviewVersion => latestPreviewVersionKey?.id as String;
+
   Version get latestPrereleaseSemanticVersion =>
       latestPrereleaseVersionKey == null
           ? null
           : Version.parse(latestPrereleaseVersion);
 
+  Version get latestPreviewSemanticVersion => latestPreviewVersionKey == null
+      ? null
+      : Version.parse(latestPreviewVersion);
+
   bool get showPrereleaseVersion {
     if (latestPrereleaseVersion == null) return false;
-    return latestSemanticVersion < latestPrereleaseSemanticVersion;
+    return latestSemanticVersion < latestPrereleaseSemanticVersion &&
+        (latestPreviewSemanticVersion == null ||
+            latestPreviewSemanticVersion < latestPrereleaseSemanticVersion);
+  }
+
+  bool get showPreviewVersion {
+    if (latestPreviewVersion == null) return false;
+    return latestSemanticVersion < latestPreviewSemanticVersion;
   }
 
   String get shortLatestPrereleasePublished {
@@ -185,22 +215,38 @@ class Package extends db.ExpandoModel<String> {
     uploaders.remove(userId);
   }
 
-  /// Updates latest stable and dev version keys with the new version.
-  void updateVersion(PackageVersion pv) {
-    final Version newVersion = pv.semanticVersion;
+  /// Updates latest stable, prerelease and preview versions and published
+  /// timestamp with the new version.
+  void updateVersion(
+    PackageVersion pv, {
+    @required Version dartSdkVersion,
+  }) {
+    final newVersion = pv.semanticVersion;
 
     if (latestVersionKey == null ||
-        isNewer(latestSemanticVersion, newVersion, pubSorted: true)) {
+        (isNewer(latestSemanticVersion, newVersion, pubSorted: true) &&
+            !pv.pubspec.isPreviewForCurrentSdk(dartSdkVersion))) {
       latestVersionKey = pv.key;
       latestPublished = pv.created;
     }
 
+    if (latestPreviewVersionKey == null ||
+        isNewer(latestPreviewSemanticVersion, newVersion, pubSorted: true)) {
+      latestPreviewVersionKey = pv.key;
+      latestPreviewPublished = pv.created;
+    }
+
     if (latestPrereleaseVersionKey == null ||
         isNewer(latestPrereleaseSemanticVersion, newVersion,
             pubSorted: false)) {
       latestPrereleaseVersionKey = pv.key;
       latestPrereleasePublished = pv.created;
     }
+
+    if (lastVersionPublished == null ||
+        lastVersionPublished.isBefore(pv.created)) {
+      lastVersionPublished = pv.created;
+    }
   }
 
   bool isNewPackage() => created.difference(DateTime.now()).abs().inDays <= 30;

app/lib/tool/test_profile/import_source.dart

@@ -111,7 +111,7 @@ class _AutoGeneratedImportSource implements ImportSource {
 
   @override
   Future<List<int>> getArchiveBytes(String package, String version) async {
-    final archive = _ArchiveBuilder();
+    final archive = ArchiveBuilder();
 
     final isFlutter = package.startsWith('flutter_');
     final pubspec = json.encode({
@@ -138,7 +138,8 @@ class _AutoGeneratedImportSource implements ImportSource {
   Future<void> close() async {}
 }
 
-class _ArchiveBuilder {
+@visibleForTesting
+class ArchiveBuilder {
   final archive = Archive();
 
   void addFile(String path, String content) {