diff --git a/drift_website/.gitignore b/drift_website/.gitignore index b7ea95fd3..cb76f5fb6 100644 --- a/drift_website/.gitignore +++ b/drift_website/.gitignore @@ -28,3 +28,5 @@ migrate_working_dir/ # VS Code which you may wish to be included in version control, so this line # is commented out by default. #.vscode/ + +web/search.db diff --git a/drift_website/lib/main.dart b/drift_website/lib/main.dart index 9990006e3..fb1c24bbe 100644 --- a/drift_website/lib/main.dart +++ b/drift_website/lib/main.dart @@ -7,22 +7,11 @@ import 'package:jaspr/server.dart'; import 'package:jaspr_content/jaspr_content.dart'; -import 'package:jaspr_content/theme.dart'; -import 'package:jaspr_content_snippets/jaspr_content_snippets.dart'; -import 'package:watcher/watcher.dart'; -import 'package:jaspr_docsy/jaspr_docsy.dart'; // This file is generated automatically by Jaspr, do not remove or edit. import 'jaspr_options.dart'; import 'src/common.dart'; -import 'src/components/collapsible.dart'; import 'src/components/inherited_page.dart'; -import 'src/components/page_ref.dart'; -import 'src/components/web_compatibility.dart'; -import 'src/generated_snippets.dart'; -import 'src/layouts/docs.dart'; -import 'src/relative_watcher.dart'; -import 'src/version_data_loader.dart'; void main() { // Initializes the server environment with the generated default options. @@ -33,57 +22,13 @@ void main() { ), ); - final fsLoader = FilesystemLoader( - 'content', - // ignore: invalid_use_of_visible_for_testing_member - watcherFactory: (path) { - final inner = DirectoryWatcher(path); - // For some reason rebuilds don't work without this, but this must - // be something I'm doing wrong. Temporary workaround until I figure - // this out. - return RelativeDirectoryWatcher(inner); - }, - ); - + final fsLoader = driftWebsiteLoader(); runApp( InheritedPageResolver( loader: fsLoader, child: ContentApp.custom( loaders: [fsLoader], - configResolver: PageConfig.all( - enableFrontmatter: true, - dataLoaders: [ - PackageVersionDataLoader(), - MemoryDataLoader({ - 'site': {'base': false}, - }), - ], - templateEngine: MustacheTemplateEngine(), - parsers: [ - MarkdownParser( - documentBuilder: (_) => driftMarkdownDocumentBuilder(), - ), - ], - extensions: [ - // Adds heading anchors to each heading. - HeadingAnchorsExtension(), - // Generates a table of contents for each page. - TableOfContentsExtension(), - ], - components: [ - PageRef(), - BetterCodeBlock(), - Collapsible.component(), - DocsyTabs(), - renderedSnippetComponent(snippets: generatedSnippets), - CustomComponent( - pattern: 'WebCompatibilityCheck', - builder: (_, _, _) => WebCompatibilityCheck(), - ), - ], - layouts: [DriftDocsLayout()], - theme: ContentTheme.none(), - ), + configResolver: driftPageConfig(), ), ), ); diff --git a/drift_website/lib/src/common.dart b/drift_website/lib/src/common.dart index 897ca2c40..1d4cdaac4 100644 --- a/drift_website/lib/src/common.dart +++ b/drift_website/lib/src/common.dart @@ -1,7 +1,19 @@ +import 'package:jaspr/src/framework/framework.dart'; import 'package:jaspr_content/jaspr_content.dart'; +import 'package:jaspr_content/theme.dart'; +import 'package:jaspr_content_snippets/jaspr_content_snippets.dart'; +import 'package:jaspr_docsy/jaspr_docsy.dart'; import 'package:markdown/markdown.dart' as md; +import 'package:watcher/watcher.dart'; import 'components/admonition.dart'; +import 'components/collapsible.dart'; +import 'components/page_ref.dart'; +import 'components/web_compatibility.dart'; +import 'generated_snippets.dart'; +import 'layouts/docs.dart'; +import 'relative_watcher.dart'; +import 'version_data_loader.dart'; md.Document driftMarkdownDocumentBuilder() { return md.Document( @@ -12,3 +24,60 @@ md.Document driftMarkdownDocumentBuilder() { extensionSet: md.ExtensionSet.gitHubWeb, ); } + +FilesystemLoader driftWebsiteLoader() { + return FilesystemLoader( + 'content', + // ignore: invalid_use_of_visible_for_testing_member + watcherFactory: (path) { + final inner = DirectoryWatcher(path); + // Workaround for https://github.com/schultek/jaspr/pull/571 + return RelativeDirectoryWatcher(inner); + }, + ); +} + +ConfigResolver driftPageConfig({bool forSearchIndex = false}) { + return PageConfig.all( + enableFrontmatter: true, + dataLoaders: [ + PackageVersionDataLoader(), + MemoryDataLoader({ + 'site': {'base': false}, + }), + ], + templateEngine: MustacheTemplateEngine(), + parsers: [ + MarkdownParser(documentBuilder: (_) => driftMarkdownDocumentBuilder()), + ], + extensions: [ + // Adds heading anchors to each heading. + HeadingAnchorsExtension(), + // Generates a table of contents for each page. + TableOfContentsExtension(), + ], + components: [ + PageRef(), + BetterCodeBlock(), + Collapsible.component(), + DocsyTabs(), + renderedSnippetComponent(snippets: generatedSnippets), + CustomComponent( + pattern: 'WebCompatibilityCheck', + builder: (_, _, _) => WebCompatibilityCheck(), + ), + ], + layouts: [if (forSearchIndex) _ContentOnlyLayout() else DriftDocsLayout()], + theme: ContentTheme.none(), + ); +} + +final class _ContentOnlyLayout extends PageLayoutBase { + @override + Component buildBody(Page page, Component child) { + return child; + } + + @override + Pattern get name => 'text'; +} diff --git a/drift_website/lib/src/components/anchors.dart b/drift_website/lib/src/components/anchors.dart index 722c64d4b..5cec0146c 100644 --- a/drift_website/lib/src/components/anchors.dart +++ b/drift_website/lib/src/components/anchors.dart @@ -1,4 +1,3 @@ -import 'package:jaspr/browser.dart'; import 'package:jaspr/jaspr.dart'; import 'package:universal_web/web.dart' as web; @@ -11,7 +10,11 @@ import 'web_compatibility_unsupported.dart' final class ActiveAnchors extends StatefulComponent { @override State createState() { - return _AnchorsState(); + if (kIsWeb) { + return _AnchorsState(); + } else { + return _ServerAnchorsState(); + } } } @@ -69,3 +72,10 @@ final class _AnchorsState extends State { return const Component.empty(); } } + +final class _ServerAnchorsState extends State { + @override + Component build(BuildContext context) { + return const Component.empty(); + } +} diff --git a/drift_website/lib/src/search/vfs.dart b/drift_website/lib/src/search/vfs.dart new file mode 100644 index 000000000..d5915f365 --- /dev/null +++ b/drift_website/lib/src/search/vfs.dart @@ -0,0 +1,307 @@ +import 'dart:async'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:http/http.dart'; +import 'package:sqlite3/common.dart'; + +final class AsyncVfsDatabase { + final CommonDatabase _db; + final HttpFileSystem _vfs; + + AsyncVfsDatabase._(this._db, this._vfs); + + static Future open( + CommonSqlite3 sqlite3, + Uri databaseUri, + ) async { + final vfs = HttpFileSystem(name: 'http', uri: databaseUri); + sqlite3.registerVirtualFileSystem(vfs); + + final db = await vfs.asyncify(() { + return sqlite3.open('/database', vfs: 'http', mode: OpenMode.readOnly); + }); + + return AsyncVfsDatabase._(db, vfs); + } + + Future _useDb(T Function() inner) async { + return _vfs.asyncify(inner); + } + + Stream search(String term) async* { + final stmt = await _useDb( + () => _db.prepare( + 'SELECT title, path, snippet(content, 1, ?2, ?3, ?4, 5) FROM content(?1) ORDER BY rank', + ), + ); + final cursor = await _useDb( + () => stmt.selectCursor([ + term, + SearchResult.startMarker, + SearchResult.endMarker, + '', + ]), + ); + + try { + while (await _useDb(cursor.moveNext)) { + final title = cursor.current.columnAt(0) as String; + final path = cursor.current.columnAt(1) as String; + final snippet = cursor.current.columnAt(2) as String; + + yield SearchResult(path, title, snippet); + } + } finally { + stmt.dispose(); + } + } + + void close() { + _db.dispose(); + _vfs._cache._client.close(); + } +} + +final class SearchResult { + final String path; + final String title; + final String highlight; + + SearchResult(this.path, this.title, this.highlight); + + static const startMarker = 'SNIPSTART'; + static const endMarker = 'SNIPEND'; +} + +final class HttpFileSystem extends BaseVirtualFileSystem { + final _BlockCache _cache; + Future? _waitingFor; + + HttpFileSystem({required super.name, required Uri uri}) + : _cache = _BlockCache(Client(), uri); + + @override + int xAccess(String path, int flags) { + return path == '/database' ? 1 : 0; + } + + @override + void xDelete(String path, int syncDir) { + throw VfsException(21); + } + + @override + String xFullPathName(String path) { + return path; + } + + @override + XOpenResult xOpen(Sqlite3Filename path, int flags) { + return (outFlags: 0, file: _HttpFile(this)); + } + + @override + void xSleep(Duration duration) {} + + /// "Synchronously" runs a potentially asynchronous operation. + /// + /// This implements a clever trick taken from Roy Hashimoto's work on + /// wa-sqlite: We're not asyncifyng SQLite itself, but we can rely on the fact + /// that SQLite statements are, essentially, coroutines that advance by + /// calling `sqlite3_step`. In a sense, SQLite is asynchronous already! + /// + /// So here, when we need to do something asynchronous like fetching blocks + /// for the search index from the remote server, we: + /// + /// 1. Return `SQLITE_BUSY` from the VFS layer. + /// 2. SQLite will forward that error to the `sqlite3_step()` invocation. + /// 3. We catch the error and await [_waitingFor], a future that + /// completes with the asynchronous operation. + /// 4. After the future has completed, we try again. + Never blockOn(Future pendingWork) { + assert(_waitingFor == null); + _waitingFor = pendingWork; + _busy(); + } + + /// The other, higher-level part of the [syncify] stack. + /// + /// This tries running a database operation and retries if it fails because + /// the VFS would like to run an asynchronous operation. + Future asyncify(T Function() operation) async { + while (true) { + try { + return operation(); + } on SqliteException catch (e) { + if (e.resultCode == 5 && _waitingFor != null) { + await _waitingFor; + _waitingFor = null; + continue; + } + + rethrow; + } + } + } + + static Never _busy() { + throw VfsException(5); // SQLITE_BUSY + } +} + +final class _HttpFile extends BaseVfsFile { + final HttpFileSystem _vfs; + + _HttpFile(this._vfs); + + @override + int readInto(Uint8List buffer, int offset) { + try { + return _tryReadInto(buffer, offset); + } on VfsException { + rethrow; + } on Object catch (e, s) { + Zone.current.handleUncaughtError(e, s); + rethrow; + } + } + + int _tryReadInto(Uint8List buffer, int offset) { + final resolvedSize = xFileSize(); + final end = min(resolvedSize, offset + buffer.length); + var remainingBytes = end - offset; + if (_vfs._cache.ensureHasRange(offset, remainingBytes) case Future f) { + return _vfs.blockOn(f); + } + + var bytesRead = 0; + while (remainingBytes > 0) { + final page = (offset + bytesRead) ~/ _BlockCache._pageSize; + final offsetInPage = (offset + bytesRead) % _BlockCache._pageSize; + final bytesToReadFromPage = min( + _BlockCache._pageSize - offsetInPage, + remainingBytes, + ); + + buffer.setRange( + bytesRead, + bytesRead + bytesToReadFromPage, + _vfs._cache._cachedPages![page]!, + offsetInPage, + ); + remainingBytes -= bytesToReadFromPage; + bytesRead += bytesToReadFromPage; + } + + return bytesRead; + } + + @override + int xCheckReservedLock() { + return 0; + } + + @override + void xClose() {} + + @override + int xFileSize() { + if (_vfs._cache.totalSize case final knownSize?) { + return knownSize; + } + + _vfs.blockOn(_vfs._cache.resolveTotalSize()); + } + + @override + void xLock(int mode) {} + + @override + void xSync(int flags) { + _unsupportedReadonly(); + } + + @override + void xTruncate(int size) { + _unsupportedReadonly(); + } + + @override + void xUnlock(int mode) {} + + @override + void xWrite(Uint8List buffer, int fileOffset) { + _unsupportedReadonly(); + } +} + +Never _unsupportedReadonly() { + throw VfsException(8); // SQLITE_READONLY +} + +final class _BlockCache { + final Client _client; + final Uri uri; + + int? totalSize; + List? _cachedPages; + + _BlockCache(this._client, this.uri); + + Future resolveTotalSize() { + return Future(() async { + final response = await _client.head(uri); + if (response.statusCode != 200) { + throw ClientException('Unexpected result code ${response.statusCode}'); + } + + final length = response.headers['content-length']; + if (length == null) { + throw ClientException('Missing content-length header'); + } + + final size = int.parse(length); + _cachedPages = List.filled((size + _pageSize - 1) ~/ _pageSize, null); + totalSize = size; + }); + } + + /// Ensures that the range from `offset` until `offset + length` (exclusive) + /// is cached. + FutureOr ensureHasRange(int offset, int length) { + var page = _pageIndex(offset); + var endPageInclusive = _pageIndex(offset + length - 1); + + for (var i = page; i <= endPageInclusive; i++) { + if (_cachedPages![page] == null) { + // We could fetch multiple pages concurrently, but most of the time + // SQLite will only read a single page at the time anyway. + return _fetchPage(i); + } + } + + return null; + } + + Future _fetchPage(int index) async { + final startOffset = index * _pageSize; + final endOffset = (index + 1) * _pageSize - 1; + final response = await _client.get( + uri, + headers: {'Range': 'bytes=$startOffset-$endOffset'}, + ); + + if (response.statusCode != 206) { + throw ClientException('Unexpected result code ${response.statusCode}'); + } + + _cachedPages![index] = response.bodyBytes; + } + + int _pageIndex(int offset) { + return offset ~/ _pageSize; + } + + static const _pageSize = 4096; +} diff --git a/drift_website/pubspec.yaml b/drift_website/pubspec.yaml index 187b9a42d..53f62a9a2 100644 --- a/drift_website/pubspec.yaml +++ b/drift_website/pubspec.yaml @@ -30,6 +30,7 @@ dependencies: jaspr_docsy: path: /home/simon/src/jaspr_tools/jaspr_docsy universal_web: ^1.1.1 + http: ^1.5.0 dev_dependencies: build_runner: ^2.6.0 diff --git a/drift_website/tool/build_search_index.dart b/drift_website/tool/build_search_index.dart new file mode 100644 index 000000000..7395b3bac --- /dev/null +++ b/drift_website/tool/build_search_index.dart @@ -0,0 +1,182 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:drift_website/src/common.dart'; +import 'package:drift_website/src/components/inherited_page.dart'; +import 'package:jaspr/server.dart'; +import 'package:jaspr/src/server/async_build_owner.dart'; +import 'package:jaspr_router/src/misc/inherited_router.dart'; +import 'package:jaspr_content/jaspr_content.dart'; +import 'package:jaspr_router/jaspr_router.dart'; +import 'package:sqlite3/sqlite3.dart'; + +/// Creates a SQLite database file containing an FTS5 table indexing all content +/// in the documentation website. +void main() async { + final loader = driftWebsiteLoader(); + final config = driftPageConfig(forSearchIndex: true); + final target = File('web/search.db'); + if (target.existsSync()) { + target.deleteSync(); + } + + final routes = await loader.loadRoutes(config, true); + + final db = sqlite3.openInMemory() + ..createContentTable() + ..execute('pragma page_size = 4096;'); + for (final route in routes) { + if (route case final Route route) { + print('Has route ${route.path}, ${route.name}, ${route.title}'); + + final binding = _RenderTextBinding(currentUrl: route.path) + ..attachRootComponent(_RenderSingleRoute(loader, route)); + final (:title, :content) = await binding.render(); + if (binding.hadError) { + print('Stopping due to error'); + exit(1); + } + + db.addPage(title, content, route.path); + } + } + + print('Created FTS5 index for pages'); + db.execute('VACUUM INTO ?', ['web/search.db']); + exit(0); + + // TODO: It would be nice to also index documentation comments. +} + +final class _RenderSingleRoute extends StatelessComponent { + final FilesystemLoader loader; + final Route route; + + _RenderSingleRoute(this.loader, this.route); + + @override + Component build(BuildContext context) { + final state = RouteState( + location: route.path, + subloc: '', + name: route.name, + ); + return InheritedRouteState( + state: state, + child: InheritedPageResolver( + child: route.builder!(context, state), + loader: loader, + ), + ); + } +} + +/// A jaspr render element implementation that just renders the inner text of +/// nodes without any escapes. +/// +/// This is used to build content for the FTS5 index. We don't want to just +/// paste raw markdown into that index because the outputs of some components, +/// in particular of ``, should also be part of the index. +final class _RenderTextBinding extends AppBinding with ComponentsBinding { + @override + final String currentUrl; + + var hadError = false; + + _RenderTextBinding({required this.currentUrl}); + + @override + RenderObject createRootRenderObject() { + return RootMarkupRenderObject(); + } + + Future<({String title, String content})> render() async { + final root = rootElement!; + final completer = Completer(); + root.binding.addPostFrameCallback(completer.complete); + await completer.future; + + final buffer = StringBuffer(); + var title = ''; + + void render( + MarkupRenderObject obj, + StringBuffer buffer, + bool strictFormatting, + ) { + if (obj case MarkupRenderElement element) { + if (element.tag == 'title') { + title = (obj.children.single as MarkupRenderText).text; + } else if (element.tag == 'style') { + // Don't render inline styles + return; + } + } + + if (obj is MarkupRenderText) { + buffer.write(obj.text); + } else { + final strictFormattingHere = + strictFormatting || + switch (obj) { + MarkupRenderElement(tag: 'span' || 'pre') => true, + _ => false, + }; + + for (final (i, child) in obj.children.indexed) { + if (!strictFormattingHere && i != 0) { + buffer.write(' '); + } + + render(child, buffer, strictFormattingHere); + } + } + } + + render(root.renderObject as MarkupRenderObject, buffer, false); + return (title: title, content: buffer.toString()); + } + + @override + BuildOwner createRootBuildOwner() { + return AsyncBuildOwner(); + } + + @override + bool get isClient => false; + + @override + void reportBuildError(Element element, Object error, StackTrace stackTrace) { + hadError = true; + stderr.writeln( + 'Error while building ${element.component.runtimeType}:\n$error\n\n$stackTrace', + ); + } + + @override + void scheduleFrame(VoidCallback frameCallback) { + throw UnsupportedError( + 'Scheduling a frame is not supported on the server, and should never happen.', + ); + } +} + +extension on Database { + void createContentTable() { + execute(''' +CREATE VIRTUAL TABLE content USING fts5( + title, + body, + path UNINDEXED +); +'''); + } + + void addPage(String title, String body, String path) { + execute('INSERT INTO content (title, body, path) VALUES (?, ?, ?)', [ + title, + body, + path, + ]); + } +} diff --git a/drift_website/tool/test_search.dart b/drift_website/tool/test_search.dart new file mode 100644 index 000000000..3c4391e05 --- /dev/null +++ b/drift_website/tool/test_search.dart @@ -0,0 +1,21 @@ +import 'package:drift_website/src/search/vfs.dart'; +import 'package:sqlite3/sqlite3.dart'; + +/// Testing the `AsyncVfsDatabase` by running a search. To test this, +/// +/// 1. Run `dart run tool/build_search_index.dart`. +/// 2. Serve web (e.g. with `dhttpd --port 8080`). +void main(List args) async { + final db = await AsyncVfsDatabase.open( + sqlite3, + Uri.parse('http://localhost:8080/search.db'), + ); + final term = 'test'; + args.join(' '); + + await for (final result in db.search(term).take(5)) { + print('${result.title}: ${result.highlight}'); + } + + db.close(); +}