Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ AppEngine version, listed here to ease deployment and troubleshooting.
## Next Release (replace with git tag when deployed)
* Bump runtimeVersion to `2025.09.08`.
* Upgraded stable Flutter analysis SDK to `3.35.3`.
* Updated SDK constraint (and language version) to `3.9`

## `20250904t074800-all`
* Bump runtimeVersion to `2025.08.29`.
Expand Down
9 changes: 6 additions & 3 deletions app/bin/tools/check_domain_access.dart
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,10 @@ Future<void> _checkHosts() async {
for (final type in [InternetAddressType.IPv4, InternetAddressType.IPv6]) {
final typeStr = type == InternetAddressType.IPv4 ? 'IPv4' : 'IPv6';
try {
final addresses =
await InternetAddress.lookup(host, type: type).timeout(timeLimit);
final addresses = await InternetAddress.lookup(
host,
type: type,
).timeout(timeLimit);
final failed = <InternetAddress>[];
for (final address in addresses) {
try {
Expand All @@ -56,7 +58,8 @@ Future<void> _checkHosts() async {
}
final successCount = addresses.length - failed.length;
print(
'- $host $typeStr resolve and connect succeeded: $successCount / ${addresses.length}');
'- $host $typeStr resolve and connect succeeded: $successCount / ${addresses.length}',
);
} catch (e) {
print('- $host $typeStr lookup failed: $e');
}
Expand Down
46 changes: 22 additions & 24 deletions app/bin/tools/check_drift.dart
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,9 @@ Future<_Sample> _sample({
Future<_Item> _singleItem(SearchForm form) async {
final params = form.toServiceQuery().toUriQueryParameters();
final rs = await _client.get(
Uri.parse('https://search-dot-dartlang-pub.appspot.com/search').replace(
queryParameters: {
...params,
'debug-drift': '1',
},
),
Uri.parse(
'https://search-dot-dartlang-pub.appspot.com/search',
).replace(queryParameters: {...params, 'debug-drift': '1'}),
);
if (rs.statusCode == 200) {
final body = json.decode(rs.body) as Map<String, dynamic>;
Expand All @@ -90,8 +87,8 @@ Future<_Item> _singleItem(SearchForm form) async {
final updatedPackages = (index['updatedPackages'] as List).cast<String>();
final lastUpdated = DateTime.parse(index['lastUpdated'] as String);

final packagesList =
(body['packages'] as List).cast<Map<String, dynamic>>();
final packagesList = (body['packages'] as List)
.cast<Map<String, dynamic>>();
final packages = <String?>[];
final scores = <String?, double?>{};

Expand Down Expand Up @@ -119,9 +116,7 @@ class _Sample {
final List<_Item>? items;
_Diff? _diff;

_Sample({
this.items,
});
_Sample({this.items});

int get length => items!.length;
_Item get first => items!.first;
Expand Down Expand Up @@ -189,17 +184,18 @@ class _Diff {
for (var j = i + 1; j < items.length; j++) {
final item = items[i];
final other = items[j];
final updatedShared = item.updatedPackages!
.toSet()
.intersection(other.updatedPackages!.toSet());
final updatedShared = item.updatedPackages!.toSet().intersection(
other.updatedPackages!.toSet(),
);
final allUpdated = <String>{
...item.updatedPackages!,
...other.updatedPackages!
...other.updatedPackages!,
};
updatedCounts.add(allUpdated.length - updatedShared.length);

final pkgShared =
item.packages!.toSet().intersection(other.packages!.toSet());
final pkgShared = item.packages!.toSet().intersection(
other.packages!.toSet(),
);
final allPackages = <String?>{...item.packages!, ...other.packages!};
packagesCounts.add(allPackages.length - pkgShared.length);

Expand All @@ -219,20 +215,22 @@ class _Diff {
updatedCount:
updatedCounts.fold<int>(0, (sum, v) => sum + v) / items.length,
updatedDuration: updatedDiffs.fold<Duration>(
Duration.zero, (mv, d) => mv > d ? mv : d),
Duration.zero,
(mv, d) => mv > d ? mv : d,
),
);
}

bool get hasDrift =>
hasObservableDifference! || packagesCount! > 0.0 || scoreDiffPct! > 0.0;

Map<String, dynamic> toJson() => {
'observable': hasObservableDifference,
'pkg': packagesCount,
'maxScore': scoreDiffPct,
'updated': updatedCount,
'maxDelta': updatedDuration,
};
'observable': hasObservableDifference,
'pkg': packagesCount,
'maxScore': scoreDiffPct,
'updated': updatedCount,
'maxDelta': updatedDuration,
};
}

class _FormWithSummary {
Expand Down
54 changes: 33 additions & 21 deletions app/bin/tools/isolate_search_benchmark.dart
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,26 @@ final queries = [
];

Future<void> main(List<String> args) async {
print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
print(
'Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB',
);
final runner = await startSearchIsolate(snapshot: args.first);
print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
print(
'Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB',
);

for (var i = 0; i < 5; i++) {
await _benchmark(runner);
print('--');
}

await runner.close();
print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
print(
'Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB',
);
}

Future<void> _benchmark(IsolateRunner primary) async {
Expand All @@ -46,25 +52,31 @@ Future<void> _benchmark(IsolateRunner primary) async {
for (var i = 0; i < 100; i++) {
final random = Random(i);
final items = queries
.map((q) => ServiceSearchQuery.parse(
query: q,
tagsPredicate: TagsPredicate.regularSearch(),
))
.map(
(q) => ServiceSearchQuery.parse(
query: q,
tagsPredicate: TagsPredicate.regularSearch(),
),
)
.toList();
items.shuffle(random);
await Future.wait(items.map((q) async {
final sw = Stopwatch()..start();
await index.search(q);
final d = sw.elapsed.inMicroseconds;
durations.putIfAbsent('all', () => []).add(d);
final key = q.parsedQuery.hasFreeText ? 'primary' : 'reduced';
durations.putIfAbsent(key, () => []).add(d);
}));
await Future.wait(
items.map((q) async {
final sw = Stopwatch()..start();
await index.search(q);
final d = sw.elapsed.inMicroseconds;
durations.putIfAbsent('all', () => []).add(d);
final key = q.parsedQuery.hasFreeText ? 'primary' : 'reduced';
durations.putIfAbsent(key, () => []).add(d);
}),
);
}
for (final e in durations.entries) {
e.value.sort();
print('${e.key.padLeft(10)}: '
'${e.value.average.round().toString().padLeft(10)} avg '
'${e.value[e.value.length * 90 ~/ 100].toString().padLeft(10)} p90');
print(
'${e.key.padLeft(10)}: '
'${e.value.average.round().toString().padLeft(10)} avg '
'${e.value[e.value.length * 90 ~/ 100].toString().padLeft(10)} p90',
);
}
}
77 changes: 41 additions & 36 deletions app/bin/tools/public_package_page_checker.dart
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,28 @@ import 'package:pool/pool.dart';
/// checks for rendering user-generated content (esp. markdown).
Future<void> main(List<String> args) async {
final argParser = ArgParser()
..addOption('concurrency',
abbr: 'c', defaultsTo: '1', help: 'Number of concurrent processing.')
..addOption(
'concurrency',
abbr: 'c',
defaultsTo: '1',
help: 'Number of concurrent processing.',
)
..addOption('pub-hosted-url', abbr: 'p', help: 'The PUB_HOSTED_URL to use.')
..addOption('limit', help: 'Stop after N successful checks.')
..addFlag('help', abbr: 'h', defaultsTo: false, help: 'Show help.');

final argv = argParser.parse(args);
if (argv['help'] as bool) {
print(
'Usage: dart public_package_page_checker.dart --pub-hosted-url https://staging-site/ -c 8');
'Usage: dart public_package_page_checker.dart --pub-hosted-url https://staging-site/ -c 8',
);
print('Crawls and checks public package pages on pub.dev site');
print(argParser.usage);
return;
}

final pubHostedUrl = (argv['pub-hosted-url'] as String?) ??
final pubHostedUrl =
(argv['pub-hosted-url'] as String?) ??
Platform.environment['PUB_HOSTED_URL'];
if (pubHostedUrl == null) {
print('Missing PUB_HOSTED_URL.');
Expand All @@ -40,48 +46,47 @@ Future<void> main(List<String> args) async {

final client = httpRetryClient();
try {
final nameRs =
await client.get(Uri.parse('$pubHostedUrl/api/package-names'));
final packages =
((json.decode(nameRs.body) as Map)['packages'] as List).cast<String>();
final nameRs = await client.get(
Uri.parse('$pubHostedUrl/api/package-names'),
);
final packages = ((json.decode(nameRs.body) as Map)['packages'] as List)
.cast<String>();
packages.shuffle();

var count = 0;
var running = true;
final pool = Pool(concurrency);
await Future.wait(
packages.map(
(p) => pool.withResource(
() async {
if (!running) {
return;
}
(p) => pool.withResource(() async {
if (!running) {
return;
}

Future<String> checkContent(String subPath) async {
final uri = Uri.parse('$pubHostedUrl/packages/$p$subPath');
print('GET $uri');
final rs = await client.get(uri);
if (rs.statusCode != 200) {
print('Failed to GET: $uri');
throw StateError('Failed to GET "$uri".');
}
return rs.body;
Future<String> checkContent(String subPath) async {
final uri = Uri.parse('$pubHostedUrl/packages/$p$subPath');
print('GET $uri');
final rs = await client.get(uri);
if (rs.statusCode != 200) {
print('Failed to GET: $uri');
throw StateError('Failed to GET "$uri".');
}
return rs.body;
}

final mainBody = await checkContent('');
if (mainBody.contains('$p/changelog')) {
await checkContent('/changelog');
}
if (mainBody.contains('$p/example')) {
await checkContent('/example');
}
count++;
if (running && limit > 0 && count >= limit) {
print('Limit reached, stopping...');
running = false;
}
},
),
final mainBody = await checkContent('');
if (mainBody.contains('$p/changelog')) {
await checkContent('/changelog');
}
if (mainBody.contains('$p/example')) {
await checkContent('/example');
}
count++;
if (running && limit > 0 && count >= limit) {
print('Limit reached, stopping...');
running = false;
}
}),
),
);
} finally {
Expand Down
26 changes: 13 additions & 13 deletions app/bin/tools/sdk_search_benchmark.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,18 @@ import 'package:pub_dev/search/sdk_mem_index.dart';

/// Loads a Dart SDK search snapshot and executes queries on it, benchmarking their total time to complete.
Future<void> main() async {
print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
print(
'Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB',
);
final index = await createSdkMemIndex();
print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
print(
'Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB',
);

// NOTE: please add more queries to this list, especially if there is a performance bottleneck.
final queries = [
'chart',
'json',
'camera',
'android camera',
'sql database',
];
final queries = ['chart', 'json', 'camera', 'android camera', 'sql database'];

final sw = Stopwatch()..start();
var count = 0;
Expand All @@ -31,6 +29,8 @@ Future<void> main() async {
}
sw.stop();
print('${(sw.elapsedMilliseconds / count).toStringAsFixed(2)} ms/request');
print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
print(
'Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB',
);
}
Loading