Skip to content

Experimental search completion #7979

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Sep 4, 2024
Merged

Experimental search completion #7979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5945d45
Experimental search completion
jonasfj Aug 26, 2024
7c451f6
Simplify a few things
jonasfj Aug 26, 2024
091a61b
Cleanup trigger logic
jonasfj Aug 26, 2024
50888de
Updated golden files
jonasfj Aug 26, 2024
c9f36cb
Updated size test
jonasfj Aug 26, 2024
7849d99
Fix tests
jonasfj Aug 27, 2024
feb8361
Merge branch 'master' of https://github.com/dart-lang/pub-dev into se…
jonasfj Sep 4, 2024
11ecb6f
Fix review comments
jonasfj Sep 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 166 additions & 0 deletions app/lib/frontend/handlers/custom_api.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:_pub_shared/data/package_api.dart';
import 'package:_pub_shared/search/search_form.dart';
import 'package:gcloud/storage.dart';
import 'package:pub_dev/frontend/templates/views/shared/search_banner.dart';
import 'package:shelf/shelf.dart' as shelf;

import '../../frontend/request_context.dart';
Expand Down Expand Up @@ -281,6 +283,170 @@ Future<shelf.Response> apiTopicNameCompletionDataHandler(
});
}

/// Handles requests for
/// - /api/search-input-completion-data
Future<shelf.Response> apiSearchInputCompletionDataHandler(
shelf.Request request,
) async {
// only accept requests which allow JSON response
if (!request.acceptsJsonContent()) {
throw NotAcceptableException(
'Client must send "Accept: application/json" header.',
);
}

final bytes = await cache.searchInputCompletionDataJsonGz().get(() async {
final topicsJson = await storageService
.bucket(activeConfiguration.reportsBucketName!)
.readAsBytes(topicsJsonFileName);
final topicsMap = json.decode(utf8.decode(topicsJson));
final topics = (topicsMap as Map<String, Object?>).keys.toList();
return gzip.encode(utf8.encode(completionDataJson(
topics: topics,
licenses: _commonLicenses,
)));
});

if (!request.acceptsGzipEncoding()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not just reject the request?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm 50/50 on that, maybe it's the better choice.

// TODO: Consider caching non-compressed response
// Note: We must handle non-gzipped response, as we can't set the
// Content-Encoding header in the browser.
// Though, all browsers will allow gzip :D
return shelf.Response(200, body: gzip.decode(bytes!), headers: {
...jsonResponseHeaders,
...CacheControl.completionData.headers
});
}

return shelf.Response(200, body: bytes, headers: {
...jsonResponseHeaders,
'Content-Encoding': 'gzip',
...CacheControl.completionData.headers
});
}

/// A hardcoded list of common licenses.
///
/// TODO: Extract a list of all licenses used from analyzed data.
const _commonLicenses = [
'0bsd',
'aal',
'afl-1.1',
'afl-1.2',
'afl-2.0',
'afl-2.1',
'afl-3.0',
'agpl-3.0',
'apl-1.0',
'apsl-1.0',
'apsl-1.1',
'apsl-1.2',
'apsl-2.0',
'apache-1.1',
'apache-2.0',
'artistic-1.0',
'artistic-1.0-perl',
'artistic-1.0-cl8',
'artistic-2.0',
'bsd-1-clause',
'bsd-2-clause',
'bsd-2-clause-patent',
'bsd-3-clause',
'bsd-3-clause-lbnl',
'bsl-1.0',
'cal-1.0-combined-work-exception',
'catosl-1.1',
'cddl-1.0',
'cecill-2.1',
'cern-ohl-p-2.0',
'cern-ohl-s-2.0',
'cern-ohl-w-2.0',
'cnri-python',
'cpal-1.0',
'cpl-1.0',
'cua-opl-1.0',
'ecl-1.0',
'ecl-2.0',
'efl-1.0',
'efl-2.0',
'epl-1.0',
'epl-2.0',
'eudatagrid',
'eupl-1.1',
'eupl-1.2',
'entessa',
'fair',
'frameworx-1.0',
'gpl-2.0',
'gpl-3.0',
'hpnd',
'ipa',
'ipl-1.0',
'isc',
'intel',
'lgpl-2.0',
'lgpl-2.1',
'lgpl-3.0',
'lpl-1.0',
'lpl-1.02',
'lppl-1.3c',
'liliq-p-1.1',
'liliq-r-1.1',
'liliq-rplus-1.1',
'mit',
'mit-0',
'mit-modern-variant',
'mpl-1.0',
'mpl-1.1',
'mpl-2.0',
'ms-pl',
'ms-rl',
'miros',
'motosoto',
'mulanpsl-2.0',
'multics',
'nasa-1.3',
'ncsa',
'ngpl',
'nposl-3.0',
'ntp',
'naumen',
'nokia',
'oclc-2.0',
'ofl-1.1',
'ogtsl',
'oldap-2.8',
'oset-pl-2.1',
'osl-1.0',
'osl-2.0',
'osl-2.1',
'osl-3.0',
'php-3.0',
'php-3.01',
'postgresql',
'python-2.0',
'qpl-1.0',
'rpl-1.1',
'rpl-1.5',
'rpsl-1.0',
'rscpl',
'sissl',
'spl-1.0',
'simpl-2.0',
'sleepycat',
'ucl-1.0',
'upl-1.0',
'unicode-dfs-2016',
'unlicense',
'vsl-1.0',
'w3c',
'watcom-1.0',
'xnet',
'zpl-2.0',
'zpl-2.1',
'zlib',
];

/// Handles requests for /api/search
Future<shelf.Response> apiSearchHandler(shelf.Request request) async {
final searchForm = SearchForm.parse(request.requestedUri.queryParameters);
Expand Down
3 changes: 3 additions & 0 deletions app/lib/frontend/handlers/experimental.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import '../../shared/cookie_utils.dart';

const _publicFlags = <String>{
'dark',
'search-completion',
};

const _allFlags = <String>{
Expand Down Expand Up @@ -83,6 +84,8 @@ class ExperimentalFlags {
return params;
}

bool get isSearchCompletionEnabled => isEnabled('search-completion');

bool get isDarkModeEnabled => isEnabled('dark');
bool get isDarkModeDefault => isEnabled('dark-as-default');

Expand Down
7 changes: 7 additions & 0 deletions app/lib/frontend/handlers/pubapi.client.dart
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions app/lib/frontend/handlers/pubapi.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,11 @@ class PubApi {
return apiTopicNameCompletionDataHandler(request);
}

@EndPoint.get('/api/search-input-completion-data')
Future<Response> searchInputCompletionData(Request request) async {
return apiSearchInputCompletionDataHandler(request);
}

@EndPoint.put('/api/packages/<package>/options')
Future<PkgOptions> setPackageOptions(
Request request, String package, PkgOptions body) =>
Expand Down
16 changes: 16 additions & 0 deletions app/lib/frontend/handlers/pubapi.g.dart
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

120 changes: 119 additions & 1 deletion app/lib/frontend/templates/views/shared/search_banner.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';

import 'package:pub_dev/frontend/request_context.dart';

import '../../../dom/dom.dart' as d;
import '../../../static_files.dart' show staticUrls;

Expand All @@ -16,17 +20,25 @@ d.Node searchBannerNode({
}) {
return d.form(
classes: ['search-bar', 'banner-item'],
attributes: {
'autocomplete': 'off',
},
action: formUrl,
children: [
d.input(
classes: ['input'],
type: 'search',
name: 'q',
placeholder: placeholder,
autocomplete: 'on',
autocomplete: 'off',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this not be conditional on the experiment?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no, we don't want the browser to auto-complete, it just suggests what you've typed before.

autofocus: autofocus,
value: queryText,
attributes: {
'title': 'Search',
if (requestContext.experimentalFlags.isSearchCompletionEnabled)
'data-widget': 'completion',
'data-completion-src': '/api/search-input-completion-data',
'data-completion-class': 'search-completion',
},
),
d.span(classes: ['icon']),
Expand Down Expand Up @@ -64,3 +76,109 @@ d.Node searchBannerNode({
],
);
}

/// Create completion data for search input.
///
/// Format is dictacted by `pkg/web_app/lib/src/completion.dart`.
///
/// If values in `match` is a prefix of what is being typed completion
/// will automatically start. It'll always try to use the longest match.
/// If not specified options available are assumed to be `terminal`, that is
/// they will be followed by whitespace.
/// If `forcedOnly` is specified, completion can only be initiated with
/// Ctrl+Space.
///
/// This can generate completion data of different sizes given [topics] and
/// [licenses].
String completionDataJson({
List<String> topics = const [],
List<String> licenses = const [],
}) =>
json.encode({
// TODO: Write a shared type for this in `pkg/_pub_shared/lib/data/`
'completions': [
{
'match': ['', '-'],
'terminal': false,
'forcedOnly': true,
'options': [
'has:',
'is:',
'license:',
'platform:',
'sdk:',
'show:',
'topic:',
'runtime:',
'dependency:',
'dependency*:',
'publisher:',
],
},
// TODO: Consider completion support for dependency:, dependency*: and publisher:
{
'match': ['is:', '-is:'],
'options': [
'dart3-compatible',
'flutter-favorite',
'legacy',
'null-safe',
'plugin',
'unlisted',
'wasm-ready',
],
},
{
'match': ['has:', '-has:'],
'options': [
'executable',
'screenshot',
],
},
{
'match': ['license:', '-license:'],
'options': [
'osi-approved',
...licenses,
],
},
{
'match': ['show:', '-show:'],
'options': [
'unlisted',
],
},
{
'match': ['sdk:', '-sdk:'],
'options': [
'dart',
'flutter',
],
},
{
'match': ['platform:', '-platform:'],
'options': [
'android',
'ios',
'linux',
'macos',
'web',
'windows',
],
},
{
'match': ['runtime:', '-runtime:'],
'options': [
'native-aot',
'native-jit',
'web',
],
},
{
'match': ['topic:', '-topic:'],
'options': [
...topics,
],
},
],
});
Loading