Start with full text search

drift_website/.gitignore

@@ -28,3 +28,5 @@ migrate_working_dir/
 # VS Code which you may wish to be included in version control, so this line
 # is commented out by default.
 #.vscode/
+
+web/search.db

drift_website/lib/main.dart

@@ -7,22 +7,11 @@
 import 'package:jaspr/server.dart';
 
 import 'package:jaspr_content/jaspr_content.dart';
-import 'package:jaspr_content/theme.dart';
-import 'package:jaspr_content_snippets/jaspr_content_snippets.dart';
-import 'package:watcher/watcher.dart';
-import 'package:jaspr_docsy/jaspr_docsy.dart';
 
 // This file is generated automatically by Jaspr, do not remove or edit.
 import 'jaspr_options.dart';
 import 'src/common.dart';
-import 'src/components/collapsible.dart';
 import 'src/components/inherited_page.dart';
-import 'src/components/page_ref.dart';
-import 'src/components/web_compatibility.dart';
-import 'src/generated_snippets.dart';
-import 'src/layouts/docs.dart';
-import 'src/relative_watcher.dart';
-import 'src/version_data_loader.dart';
 
 void main() {
   // Initializes the server environment with the generated default options.
@@ -33,57 +22,13 @@ void main() {
     ),
   );
 
-  final fsLoader = FilesystemLoader(
-    'content',
-    // ignore: invalid_use_of_visible_for_testing_member
-    watcherFactory: (path) {
-      final inner = DirectoryWatcher(path);
-      // For some reason rebuilds don't work without this, but this must
-      // be something I'm doing wrong. Temporary workaround until I figure
-      // this out.
-      return RelativeDirectoryWatcher(inner);
-    },
-  );
-
+  final fsLoader = driftWebsiteLoader();
   runApp(
     InheritedPageResolver(
       loader: fsLoader,
       child: ContentApp.custom(
         loaders: [fsLoader],
-        configResolver: PageConfig.all(
-          enableFrontmatter: true,
-          dataLoaders: [
-            PackageVersionDataLoader(),
-            MemoryDataLoader({
-              'site': {'base': false},
-            }),
-          ],
-          templateEngine: MustacheTemplateEngine(),
-          parsers: [
-            MarkdownParser(
-              documentBuilder: (_) => driftMarkdownDocumentBuilder(),
-            ),
-          ],
-          extensions: [
-            // Adds heading anchors to each heading.
-            HeadingAnchorsExtension(),
-            // Generates a table of contents for each page.
-            TableOfContentsExtension(),
-          ],
-          components: [
-            PageRef(),
-            BetterCodeBlock(),
-            Collapsible.component(),
-            DocsyTabs(),
-            renderedSnippetComponent(snippets: generatedSnippets),
-            CustomComponent(
-              pattern: 'WebCompatibilityCheck',
-              builder: (_, _, _) => WebCompatibilityCheck(),
-            ),
-          ],
-          layouts: [DriftDocsLayout()],
-          theme: ContentTheme.none(),
-        ),
+        configResolver: driftPageConfig(),
       ),
     ),
   );

drift_website/lib/src/common.dart

@@ -1,7 +1,19 @@
+import 'package:jaspr/src/framework/framework.dart';
 import 'package:jaspr_content/jaspr_content.dart';
+import 'package:jaspr_content/theme.dart';
+import 'package:jaspr_content_snippets/jaspr_content_snippets.dart';
+import 'package:jaspr_docsy/jaspr_docsy.dart';
 import 'package:markdown/markdown.dart' as md;
+import 'package:watcher/watcher.dart';
 
 import 'components/admonition.dart';
+import 'components/collapsible.dart';
+import 'components/page_ref.dart';
+import 'components/web_compatibility.dart';
+import 'generated_snippets.dart';
+import 'layouts/docs.dart';
+import 'relative_watcher.dart';
+import 'version_data_loader.dart';
 
 md.Document driftMarkdownDocumentBuilder() {
   return md.Document(
@@ -12,3 +24,60 @@ md.Document driftMarkdownDocumentBuilder() {
     extensionSet: md.ExtensionSet.gitHubWeb,
   );
 }
+
+FilesystemLoader driftWebsiteLoader() {
+  return FilesystemLoader(
+    'content',
+    // ignore: invalid_use_of_visible_for_testing_member
+    watcherFactory: (path) {
+      final inner = DirectoryWatcher(path);
+      // Workaround for https://github.com/schultek/jaspr/pull/571
+      return RelativeDirectoryWatcher(inner);
+    },
+  );
+}
+
+ConfigResolver driftPageConfig({bool forSearchIndex = false}) {
+  return PageConfig.all(
+    enableFrontmatter: true,
+    dataLoaders: [
+      PackageVersionDataLoader(),
+      MemoryDataLoader({
+        'site': {'base': false},
+      }),
+    ],
+    templateEngine: MustacheTemplateEngine(),
+    parsers: [
+      MarkdownParser(documentBuilder: (_) => driftMarkdownDocumentBuilder()),
+    ],
+    extensions: [
+      // Adds heading anchors to each heading.
+      HeadingAnchorsExtension(),
+      // Generates a table of contents for each page.
+      TableOfContentsExtension(),
+    ],
+    components: [
+      PageRef(),
+      BetterCodeBlock(),
+      Collapsible.component(),
+      DocsyTabs(),
+      renderedSnippetComponent(snippets: generatedSnippets),
+      CustomComponent(
+        pattern: 'WebCompatibilityCheck',
+        builder: (_, _, _) => WebCompatibilityCheck(),
+      ),
+    ],
+    layouts: [if (forSearchIndex) _ContentOnlyLayout() else DriftDocsLayout()],
+    theme: ContentTheme.none(),
+  );
+}
+
+final class _ContentOnlyLayout extends PageLayoutBase {
+  @override
+  Component buildBody(Page page, Component child) {
+    return child;
+  }
+
+  @override
+  Pattern get name => 'text';
+}

drift_website/lib/src/components/anchors.dart

@@ -1,4 +1,3 @@
-import 'package:jaspr/browser.dart';
 import 'package:jaspr/jaspr.dart';
 import 'package:universal_web/web.dart' as web;
 
@@ -11,7 +10,11 @@ import 'web_compatibility_unsupported.dart'
 final class ActiveAnchors extends StatefulComponent {
   @override
   State<StatefulComponent> createState() {
-    return _AnchorsState();
+    if (kIsWeb) {
+      return _AnchorsState();
+    } else {
+      return _ServerAnchorsState();
+    }
   }
 }
 
@@ -69,3 +72,10 @@ final class _AnchorsState extends State<ActiveAnchors> {
     return const Component.empty();
   }
 }
+
+final class _ServerAnchorsState extends State<ActiveAnchors> {
+  @override
+  Component build(BuildContext context) {
+    return const Component.empty();
+  }
+}

drift_website/lib/src/search/vfs.dart

@@ -0,0 +1,307 @@
+import 'dart:async';
+import 'dart:math';
+import 'dart:typed_data';
+
+import 'package:http/http.dart';
+import 'package:sqlite3/common.dart';
+
+final class AsyncVfsDatabase {
+  final CommonDatabase _db;
+  final HttpFileSystem _vfs;
+
+  AsyncVfsDatabase._(this._db, this._vfs);
+
+  static Future<AsyncVfsDatabase> open(
+    CommonSqlite3 sqlite3,
+    Uri databaseUri,
+  ) async {
+    final vfs = HttpFileSystem(name: 'http', uri: databaseUri);
+    sqlite3.registerVirtualFileSystem(vfs);
+
+    final db = await vfs.asyncify(() {
+      return sqlite3.open('/database', vfs: 'http', mode: OpenMode.readOnly);
+    });
+
+    return AsyncVfsDatabase._(db, vfs);
+  }
+
+  Future<T> _useDb<T>(T Function() inner) async {
+    return _vfs.asyncify(inner);
+  }
+
+  Stream<SearchResult> search(String term) async* {
+    final stmt = await _useDb(
+      () => _db.prepare(
+        'SELECT title, path, snippet(content, 1, ?2, ?3, ?4, 5) FROM content(?1) ORDER BY rank',
+      ),
+    );
+    final cursor = await _useDb(
+      () => stmt.selectCursor([
+        term,
+        SearchResult.startMarker,
+        SearchResult.endMarker,
+        '',
+      ]),
+    );
+
+    try {
+      while (await _useDb(cursor.moveNext)) {
+        final title = cursor.current.columnAt(0) as String;
+        final path = cursor.current.columnAt(1) as String;
+        final snippet = cursor.current.columnAt(2) as String;
+
+        yield SearchResult(path, title, snippet);
+      }
+    } finally {
+      stmt.dispose();
+    }
+  }
+
+  void close() {
+    _db.dispose();
+    _vfs._cache._client.close();
+  }
+}
+
+final class SearchResult {
+  final String path;
+  final String title;
+  final String highlight;
+
+  SearchResult(this.path, this.title, this.highlight);
+
+  static const startMarker = 'SNIPSTART';
+  static const endMarker = 'SNIPEND';
+}
+
+final class HttpFileSystem extends BaseVirtualFileSystem {
+  final _BlockCache _cache;
+  Future<void>? _waitingFor;
+
+  HttpFileSystem({required super.name, required Uri uri})
+    : _cache = _BlockCache(Client(), uri);
+
+  @override
+  int xAccess(String path, int flags) {
+    return path == '/database' ? 1 : 0;
+  }
+
+  @override
+  void xDelete(String path, int syncDir) {
+    throw VfsException(21);
+  }
+
+  @override
+  String xFullPathName(String path) {
+    return path;
+  }
+
+  @override
+  XOpenResult xOpen(Sqlite3Filename path, int flags) {
+    return (outFlags: 0, file: _HttpFile(this));
+  }
+
+  @override
+  void xSleep(Duration duration) {}
+
+  /// "Synchronously" runs a potentially asynchronous operation.
+  ///
+  /// This implements a clever trick taken from Roy Hashimoto's work on
+  /// wa-sqlite: We're not asyncifyng SQLite itself, but we can rely on the fact
+  /// that SQLite statements are, essentially, coroutines that advance by
+  /// calling `sqlite3_step`. In a sense, SQLite is asynchronous already!
+  ///
+  /// So here, when we need to do something asynchronous like fetching blocks
+  /// for the search index from the remote server, we:
+  ///
+  ///   1. Return `SQLITE_BUSY` from the VFS layer.
+  ///   2. SQLite will forward that error to the `sqlite3_step()` invocation.
+  ///   3. We catch the error and await [_waitingFor], a future that
+  ///      completes with the asynchronous operation.
+  ///   4. After the future has completed, we try again.
+  Never blockOn(Future<void> pendingWork) {
+    assert(_waitingFor == null);
+    _waitingFor = pendingWork;
+    _busy();
+  }
+
+  /// The other, higher-level part of the [syncify] stack.
+  ///
+  /// This tries running a database operation and retries if it fails because
+  /// the VFS would like to run an asynchronous operation.
+  Future<T> asyncify<T>(T Function() operation) async {
+    while (true) {
+      try {
+        return operation();
+      } on SqliteException catch (e) {
+        if (e.resultCode == 5 && _waitingFor != null) {
+          await _waitingFor;
+          _waitingFor = null;
+          continue;
+        }
+
+        rethrow;
+      }
+    }
+  }
+
+  static Never _busy() {
+    throw VfsException(5); // SQLITE_BUSY
+  }
+}
+
+final class _HttpFile extends BaseVfsFile {
+  final HttpFileSystem _vfs;
+
+  _HttpFile(this._vfs);
+
+  @override
+  int readInto(Uint8List buffer, int offset) {
+    try {
+      return _tryReadInto(buffer, offset);
+    } on VfsException {
+      rethrow;
+    } on Object catch (e, s) {
+      Zone.current.handleUncaughtError(e, s);
+      rethrow;
+    }
+  }
+
+  int _tryReadInto(Uint8List buffer, int offset) {
+    final resolvedSize = xFileSize();
+    final end = min(resolvedSize, offset + buffer.length);
+    var remainingBytes = end - offset;
+    if (_vfs._cache.ensureHasRange(offset, remainingBytes) case Future f) {
+      return _vfs.blockOn(f);
+    }
+
+    var bytesRead = 0;
+    while (remainingBytes > 0) {
+      final page = (offset + bytesRead) ~/ _BlockCache._pageSize;
+      final offsetInPage = (offset + bytesRead) % _BlockCache._pageSize;
+      final bytesToReadFromPage = min(
+        _BlockCache._pageSize - offsetInPage,
+        remainingBytes,
+      );
+
+      buffer.setRange(
+        bytesRead,
+        bytesRead + bytesToReadFromPage,
+        _vfs._cache._cachedPages![page]!,
+        offsetInPage,
+      );
+      remainingBytes -= bytesToReadFromPage;
+      bytesRead += bytesToReadFromPage;
+    }
+
+    return bytesRead;
+  }
+
+  @override
+  int xCheckReservedLock() {
+    return 0;
+  }
+
+  @override
+  void xClose() {}
+
+  @override
+  int xFileSize() {
+    if (_vfs._cache.totalSize case final knownSize?) {
+      return knownSize;
+    }
+
+    _vfs.blockOn(_vfs._cache.resolveTotalSize());
+  }
+
+  @override
+  void xLock(int mode) {}
+
+  @override
+  void xSync(int flags) {
+    _unsupportedReadonly();
+  }
+
+  @override
+  void xTruncate(int size) {
+    _unsupportedReadonly();
+  }
+
+  @override
+  void xUnlock(int mode) {}
+
+  @override
+  void xWrite(Uint8List buffer, int fileOffset) {
+    _unsupportedReadonly();
+  }
+}
+
+Never _unsupportedReadonly() {
+  throw VfsException(8); // SQLITE_READONLY
+}
+
+final class _BlockCache {
+  final Client _client;
+  final Uri uri;
+
+  int? totalSize;
+  List<Uint8List?>? _cachedPages;
+
+  _BlockCache(this._client, this.uri);
+
+  Future<void> resolveTotalSize() {
+    return Future(() async {
+      final response = await _client.head(uri);
+      if (response.statusCode != 200) {
+        throw ClientException('Unexpected result code ${response.statusCode}');
+      }
+
+      final length = response.headers['content-length'];
+      if (length == null) {
+        throw ClientException('Missing content-length header');
+      }
+
+      final size = int.parse(length);
+      _cachedPages = List.filled((size + _pageSize - 1) ~/ _pageSize, null);
+      totalSize = size;
+    });
+  }
+
+  /// Ensures that the range from `offset` until `offset + length` (exclusive)
+  /// is cached.
+  FutureOr<void> ensureHasRange(int offset, int length) {
+    var page = _pageIndex(offset);
+    var endPageInclusive = _pageIndex(offset + length - 1);
+
+    for (var i = page; i <= endPageInclusive; i++) {
+      if (_cachedPages![page] == null) {
+        // We could fetch multiple pages concurrently, but most of the time
+        // SQLite will only read a single page at the time anyway.
+        return _fetchPage(i);
+      }
+    }
+
+    return null;
+  }
+
+  Future<void> _fetchPage(int index) async {
+    final startOffset = index * _pageSize;
+    final endOffset = (index + 1) * _pageSize - 1;
+    final response = await _client.get(
+      uri,
+      headers: {'Range': 'bytes=$startOffset-$endOffset'},
+    );
+
+    if (response.statusCode != 206) {
+      throw ClientException('Unexpected result code ${response.statusCode}');
+    }
+
+    _cachedPages![index] = response.bodyBytes;
+  }
+
+  int _pageIndex(int offset) {
+    return offset ~/ _pageSize;
+  }
+
+  static const _pageSize = 4096;
+}

drift_website/pubspec.yaml

@@ -30,6 +30,7 @@ dependencies:
   jaspr_docsy:
     path: /home/simon/src/jaspr_tools/jaspr_docsy
   universal_web: ^1.1.1
+  http: ^1.5.0
 
 dev_dependencies:
   build_runner: ^2.6.0

drift_website/tool/build_search_index.dart

@@ -0,0 +1,182 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:drift_website/src/common.dart';
+import 'package:drift_website/src/components/inherited_page.dart';
+import 'package:jaspr/server.dart';
+import 'package:jaspr/src/server/async_build_owner.dart';
+import 'package:jaspr_router/src/misc/inherited_router.dart';
+import 'package:jaspr_content/jaspr_content.dart';
+import 'package:jaspr_router/jaspr_router.dart';
+import 'package:sqlite3/sqlite3.dart';
+
+/// Creates a SQLite database file containing an FTS5 table indexing all content
+/// in the documentation website.
+void main() async {
+  final loader = driftWebsiteLoader();
+  final config = driftPageConfig(forSearchIndex: true);
+  final target = File('web/search.db');
+  if (target.existsSync()) {
+    target.deleteSync();
+  }
+
+  final routes = await loader.loadRoutes(config, true);
+
+  final db = sqlite3.openInMemory()
+    ..createContentTable()
+    ..execute('pragma page_size = 4096;');
+  for (final route in routes) {
+    if (route case final Route route) {
+      print('Has route ${route.path}, ${route.name}, ${route.title}');
+
+      final binding = _RenderTextBinding(currentUrl: route.path)
+        ..attachRootComponent(_RenderSingleRoute(loader, route));
+      final (:title, :content) = await binding.render();
+      if (binding.hadError) {
+        print('Stopping due to error');
+        exit(1);
+      }
+
+      db.addPage(title, content, route.path);
+    }
+  }
+
+  print('Created FTS5 index for pages');
+  db.execute('VACUUM INTO ?', ['web/search.db']);
+  exit(0);
+
+  // TODO: It would be nice to also index documentation comments.
+}
+
+final class _RenderSingleRoute extends StatelessComponent {
+  final FilesystemLoader loader;
+  final Route route;
+
+  _RenderSingleRoute(this.loader, this.route);
+
+  @override
+  Component build(BuildContext context) {
+    final state = RouteState(
+      location: route.path,
+      subloc: '',
+      name: route.name,
+    );
+    return InheritedRouteState(
+      state: state,
+      child: InheritedPageResolver(
+        child: route.builder!(context, state),
+        loader: loader,
+      ),
+    );
+  }
+}
+
+/// A jaspr render element implementation that just renders the inner text of
+/// nodes without any escapes.
+///
+/// This is used to build content for the FTS5 index. We don't want to just
+/// paste raw markdown into that index because the outputs of some components,
+/// in particular of `<Snippets />`, should also be part of the index.
+final class _RenderTextBinding extends AppBinding with ComponentsBinding {
+  @override
+  final String currentUrl;
+
+  var hadError = false;
+
+  _RenderTextBinding({required this.currentUrl});
+
+  @override
+  RenderObject createRootRenderObject() {
+    return RootMarkupRenderObject();
+  }
+
+  Future<({String title, String content})> render() async {
+    final root = rootElement!;
+    final completer = Completer();
+    root.binding.addPostFrameCallback(completer.complete);
+    await completer.future;
+
+    final buffer = StringBuffer();
+    var title = '';
+
+    void render(
+      MarkupRenderObject obj,
+      StringBuffer buffer,
+      bool strictFormatting,
+    ) {
+      if (obj case MarkupRenderElement element) {
+        if (element.tag == 'title') {
+          title = (obj.children.single as MarkupRenderText).text;
+        } else if (element.tag == 'style') {
+          // Don't render inline styles
+          return;
+        }
+      }
+
+      if (obj is MarkupRenderText) {
+        buffer.write(obj.text);
+      } else {
+        final strictFormattingHere =
+            strictFormatting ||
+            switch (obj) {
+              MarkupRenderElement(tag: 'span' || 'pre') => true,
+              _ => false,
+            };
+
+        for (final (i, child) in obj.children.indexed) {
+          if (!strictFormattingHere && i != 0) {
+            buffer.write(' ');
+          }
+
+          render(child, buffer, strictFormattingHere);
+        }
+      }
+    }
+
+    render(root.renderObject as MarkupRenderObject, buffer, false);
+    return (title: title, content: buffer.toString());
+  }
+
+  @override
+  BuildOwner createRootBuildOwner() {
+    return AsyncBuildOwner();
+  }
+
+  @override
+  bool get isClient => false;
+
+  @override
+  void reportBuildError(Element element, Object error, StackTrace stackTrace) {
+    hadError = true;
+    stderr.writeln(
+      'Error while building ${element.component.runtimeType}:\n$error\n\n$stackTrace',
+    );
+  }
+
+  @override
+  void scheduleFrame(VoidCallback frameCallback) {
+    throw UnsupportedError(
+      'Scheduling a frame is not supported on the server, and should never happen.',
+    );
+  }
+}
+
+extension on Database {
+  void createContentTable() {
+    execute('''
+CREATE VIRTUAL TABLE content USING fts5(
+  title,
+  body,
+  path UNINDEXED
+);
+''');
+  }
+
+  void addPage(String title, String body, String path) {
+    execute('INSERT INTO content (title, body, path) VALUES (?, ?, ?)', [
+      title,
+      body,
+      path,
+    ]);
+  }
+}

drift_website/tool/test_search.dart

@@ -0,0 +1,21 @@
+import 'package:drift_website/src/search/vfs.dart';
+import 'package:sqlite3/sqlite3.dart';
+
+/// Testing the `AsyncVfsDatabase` by running a search. To test this,
+///
+///  1. Run `dart run tool/build_search_index.dart`.
+///  2. Serve web (e.g. with `dhttpd --port 8080`).
+void main(List<String> args) async {
+  final db = await AsyncVfsDatabase.open(
+    sqlite3,
+    Uri.parse('http://localhost:8080/search.db'),
+  );
+  final term = 'test';
+  args.join(' ');
+
+  await for (final result in db.search(term).take(5)) {
+    print('${result.title}: ${result.highlight}');
+  }
+
+  db.close();
+}