From 08325008c41eb503c631b35813bb282fff932de4 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Sat, 18 Nov 2023 23:41:14 -0800 Subject: [PATCH 1/4] devops: use `pkgconfig` to resolve dependencies This way `sqlite` and `libmicrohttpd` could be effortlessly installed on macos via Homebrew, and the compilation will just work. NOTE: there's also a local `sqlite` installation on MacOS; to disambiguate with it, use `PKG_CONFIG_PATH`: ```bash export PKG_CONFIG_PATH=/opt/homebrew/Cellar/sqlite/3.44.0/lib/pkgconfig/ ``` --- CMakeLists.txt | 11 +++++++++++ Dockerfile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1012ab..701cf01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,17 @@ project(sqlite_web_vfs VERSION 1.0 DESCRIPTION "SQLite3 extension for read-only HTTP(S) database access" LANGUAGES C CXX) +# Enable pkg-config support in CMake +find_package(PkgConfig REQUIRED) + +# Find SQLite and libmicrohttpd using pkg-config +pkg_check_modules(SQLITE REQUIRED sqlite3) +pkg_check_modules(LIBMHD REQUIRED libmicrohttpd) + +include_directories(${SQLITE_INCLUDE_DIRS} ${LIBMHD_INCLUDE_DIRS}) +link_directories(${SQLITE_LIBRARY_DIRS} ${LIBMHD_LIBRARY_DIRS}) + + set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/Dockerfile b/Dockerfile index 7b40069..b162fa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND noninteractive ARG build_type=Release RUN apt-get -qq update && apt-get -qq install -y --no-install-recommends --no-install-suggests \ - ca-certificates git-core build-essential cmake \ + ca-certificates git-core build-essential cmake pkg-config \ libcurl4-openssl-dev sqlite3 libsqlite3-dev libmicrohttpd-dev \ python3-pytest pylint aria2 From be5715f168e5776a8676a4348e83b453a3034d76 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Mon, 20 Nov 2023 19:15:55 -0800 Subject: [PATCH 2/4] chore: publish as an NPM package The build script will build package when installed. --- .gitignore | 2 ++ build.sh | 26 ++++++++++++++++++++++++++ index.d.ts | 3 +++ index.mjs | 20 ++++++++++++++++++++ package-lock.json | 15 +++++++++++++++ package.json | 15 +++++++++++++++ 6 files changed, 81 insertions(+) create mode 100755 build.sh create mode 100644 index.d.ts create mode 100644 index.mjs create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 5833656..31837ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ build/ +out/ +node_modules/ .vscode/ **/__pycache__ diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..18c91a0 --- /dev/null +++ b/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e +set +x + +trap "cd $(pwd -P)" EXIT +cd "$(dirname "$0")" + +if [[ $(uname) == "Darwin" ]]; then + # Make sure pkg-config picks up dependencies from HomeBrew + export PKG_CONFIG_PATH='/opt/homebrew/Cellar/sqlite/3.44.0/lib/pkgconfig/' +fi + +rm -rf ./build +rm -rf ./out + +cmake -DCMAKE_BUILD_TYPE=Release -B build . +cmake --build build -j8 +mkdir ./out + +if [[ $(uname) == "Darwin" ]]; then + cp ./build/web_vfs.dylib ./out/web_vfs.dylib +else + cp ./build/web_vfs.so ./out/web_vfs.so +fi + +rm -rf ./build diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..bb620f0 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,3 @@ +import type { PathLike } from 'fs'; + +export function getWebVFSExtensionPath(): PathLike; diff --git a/index.mjs b/index.mjs new file mode 100644 index 0000000..67c9860 --- /dev/null +++ b/index.mjs @@ -0,0 +1,20 @@ +import { fileURLToPath } from 'url'; +import path from 'path'; +import url from 'url'; +import fs from 'fs'; + +const __filename = url.fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const LIBRARY_PATH = { + darwin: path.join(__dirname, 'out', 'web_vfs.dylib'), + linux: path.join(__dirname, 'out', 'web_vfs.so'), +}[process.platform]; + +export function getWebVFSExtensionPath() { + if (!LIBRARY_PATH) + throw new Error(`Platform "${process.platform}" is not supported`); + if (!fs.existsSync(LIBRARY_PATH)) + throw new Error(`Extension does not exist at "${LIBRARY_PATH}"`); + return LIBRARY_PATH; +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..4a4428e --- /dev/null +++ b/package-lock.json @@ -0,0 +1,15 @@ +{ + "name": "@degulabs/sqlite_web_vfs", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@degulabs/sqlite_web_vfs", + "version": "0.0.1", + "hasInstallScript": true, + "license": "ISC", + "devDependencies": {} + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..a4aafa0 --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "@degulabs/sqlite_web_vfs", + "version": "0.0.1", + "description": "This [SQLite3 virtual filesystem extension](https://www.sqlite.org/vfs.html) provides read-only access to database files over HTTP(S), including S3 and the like, without involving a [FUSE mount](https://en.wikipedia.org/wiki/Filesystem_in_Userspace) (a fine alternative when available). **See also** the companion projects [sqlite_zstd_vfs](https://github.com/mlin/sqlite_zstd_vfs/) and [Genomics Extension for SQLite](https://github.com/mlin/GenomicSQLite), which include sqlite_web_vfs along with other features, most notably compression of the database file.", + "main": "index.mjs", + "directories": { + "test": "test" + }, + "scripts": { + "install": "./build.sh" + }, + "keywords": [], + "author": "", + "license": "ISC" +} From 0f7f2eafbc2b0ebba238f85580d6200afbef6a2c Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Mon, 20 Nov 2023 19:17:41 -0800 Subject: [PATCH 3/4] docs: add clarification note --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3155c73..aed66f3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +> NOTE: this is a fork that provides this extension in a form of NPM package. +> The extension will be compiled as part of the package install script. +> It also provides a function to get the compiled extension path, see `index.mjs`. + # sqlite_web_vfs This [SQLite3 virtual filesystem extension](https://www.sqlite.org/vfs.html) provides read-only access to database files over HTTP(S), including S3 and the like, without involving a [FUSE mount](https://en.wikipedia.org/wiki/Filesystem_in_Userspace) (a fine alternative when available). **See also** the companion projects [sqlite_zstd_vfs](https://github.com/mlin/sqlite_zstd_vfs/) and [Genomics Extension for SQLite](https://github.com/mlin/GenomicSQLite), which include sqlite_web_vfs along with other features, most notably compression of the database file. From 428ae0fbc8e8846965d96da9975f81caae8e1751 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Wed, 22 Nov 2023 17:15:59 -0800 Subject: [PATCH 4/4] chore: fix types --- index.d.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/index.d.ts b/index.d.ts index bb620f0..17ccfb6 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,3 +1 @@ -import type { PathLike } from 'fs'; - -export function getWebVFSExtensionPath(): PathLike; +export function getWebVFSExtensionPath(): string;