From a39af1a829ba8de2b64c2cd82f32062db88dea9c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 7 May 2026 02:57:59 +0800 Subject: [PATCH] refactor: use modernc sqlite driver as default (#37562) The mattn driver is still kept, can be enabled by TAGS="sqlite_mattn sqlite_unlock_notify" --------- Co-authored-by: TheFox0x7 --- .github/workflows/cache-seeder.yml | 9 ++--- .github/workflows/pull-compliance.yml | 6 +-- .github/workflows/pull-db-tests.yml | 13 +++--- .github/workflows/pull-e2e-tests.yml | 4 +- .github/workflows/release-nightly.yml | 2 +- .github/workflows/release-tag-rc.yml | 2 +- .github/workflows/release-tag-version.yml | 2 +- AGENTS.md | 2 +- Dockerfile | 2 +- Dockerfile.rootless | 2 +- Makefile | 5 +-- README.md | 4 -- README.zh-cn.md | 4 -- README.zh-tw.md | 4 -- assets/go-licenses.json | 25 ++++++++++++ contrib/ide/vscode/launch.json | 13 ------ contrib/ide/vscode/settings.json | 4 +- contrib/ide/vscode/tasks.json | 22 ----------- custom/conf/app.example.ini | 2 +- flake.nix | 2 +- go.mod | 6 +++ go.sum | 48 +++++++++++++---------- models/db/conn.go | 20 ++++++++-- models/db/driver_sqlite_mattn.go | 9 ++--- models/db/driver_sqlite_modernc.go | 41 +++++++++++++++++++ models/issues/issue_stats.go | 10 ++--- models/issues/issue_test.go | 20 +++------- models/unittest/fixtures_test.go | 4 +- models/unittest/testdb.go | 8 ++-- modules/setting/database.go | 19 ++++++--- options/locale/locale_en-US.json | 1 - routers/install/install.go | 11 ++---- snap/snapcraft.yaml | 2 +- tests/integration/api_issue_test.go | 6 +-- 34 files changed, 185 insertions(+), 149 deletions(-) create mode 100644 models/db/driver_sqlite_modernc.go diff --git a/.github/workflows/cache-seeder.yml b/.github/workflows/cache-seeder.yml index cd086fae17..b0f3ca97de 100644 --- a/.github/workflows/cache-seeder.yml +++ b/.github/workflows/cache-seeder.yml @@ -45,8 +45,7 @@ jobs: cache-name: seed - run: make deps-backend - run: TAGS="bindata" make backend - - run: TAGS="bindata sqlite sqlite_unlock_notify" make backend - - run: TAGS="bindata gogit sqlite sqlite_unlock_notify" GOEXPERIMENT="" make backend + - run: TAGS="bindata gogit" GOEXPERIMENT="" make backend lint: runs-on: ubuntu-latest @@ -54,9 +53,9 @@ jobs: fail-fast: false matrix: include: - - { job: lint-backend, tags: "bindata sqlite sqlite_unlock_notify", target: "lint-backend" } - - { job: lint-go-windows, tags: "bindata sqlite sqlite_unlock_notify", target: "lint-go-windows" } - - { job: lint-go-gogit, tags: "bindata gogit sqlite sqlite_unlock_notify", target: "lint-go" } + - { job: lint-backend, tags: "bindata", target: "lint-backend" } + - { job: lint-go-windows, tags: "bindata", target: "lint-go-windows" } + - { job: lint-go-gogit, tags: "bindata gogit", target: "lint-go" } steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml index dc46c9fa42..3d7e9f8be8 100644 --- a/.github/workflows/pull-compliance.yml +++ b/.github/workflows/pull-compliance.yml @@ -32,7 +32,7 @@ jobs: - run: make deps-backend deps-tools - run: make lint-backend env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata lint-on-demand: needs: files-changed @@ -82,7 +82,7 @@ jobs: - run: make deps-backend deps-tools - run: make lint-go-windows env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata GOOS: windows GOARCH: amd64 @@ -104,7 +104,7 @@ jobs: - run: make deps-backend deps-tools - run: make lint-go env: - TAGS: bindata gogit sqlite sqlite_unlock_notify + TAGS: bindata gogit checks-backend: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 944a117e15..b750a9c264 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -62,6 +62,8 @@ jobs: run: GITEA_TEST_DATABASE=pgsql make test-integration timeout-minutes: 50 env: + # pgsql is chosen to be the unlucky one to run with the slow "race detector", it is about 60% slower. + GOTEST_FLAGS: -race -timeout=40m TAGS: bindata gogit TEST_LDAP: 1 @@ -82,7 +84,7 @@ jobs: - run: make deps-backend - run: make backend env: - TAGS: bindata gogit sqlite sqlite_unlock_notify + TAGS: bindata gogit GOEXPERIMENT: - name: run migration tests run: GITEA_TEST_DATABASE=sqlite make test-migration @@ -92,9 +94,10 @@ jobs: run: GITEA_TEST_DATABASE=sqlite make test-integration timeout-minutes: 50 env: + # sqlite driver can contain large amount of Golang code, so don't use race detector for it, otherwise, extremely slow + GOTEST_FLAGS: -timeout=40m TAGS: bindata gogit GOEXPERIMENT: - GOTEST_FLAGS: -race -timeout=40m test-unit: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' @@ -154,15 +157,15 @@ jobs: - name: unit-tests run: make test-backend test-check env: - TAGS: bindata sqlite sqlite_unlock_notify GOTEST_FLAGS: -race -timeout=20m + TAGS: bindata GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - name: unit-tests-gogit run: make test-backend test-check env: - TAGS: bindata gogit sqlite sqlite_unlock_notify - GOEXPERIMENT: GOTEST_FLAGS: -race -timeout=20m + TAGS: bindata gogit + GOEXPERIMENT: GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} test-mysql: diff --git a/.github/workflows/pull-e2e-tests.yml b/.github/workflows/pull-e2e-tests.yml index 974d82ea5e..6743e6c37a 100644 --- a/.github/workflows/pull-e2e-tests.yml +++ b/.github/workflows/pull-e2e-tests.yml @@ -40,11 +40,11 @@ jobs: - run: make deps-backend - run: make backend env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata - run: make playwright - run: make test-e2e timeout-minutes: 10 env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata FORCE_COLOR: 1 GITEA_TEST_E2E_DEBUG: 1 diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index a5fa452ef3..ee5c4fcfb6 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -32,7 +32,7 @@ jobs: # xgo build - run: make release env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata - name: import gpg key id: import_gpg uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7.0.0 diff --git a/.github/workflows/release-tag-rc.yml b/.github/workflows/release-tag-rc.yml index 2e0f2dd5c0..05b56b6a2b 100644 --- a/.github/workflows/release-tag-rc.yml +++ b/.github/workflows/release-tag-rc.yml @@ -33,7 +33,7 @@ jobs: # xgo build - run: make release env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata - name: import gpg key id: import_gpg uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7.0.0 diff --git a/.github/workflows/release-tag-version.yml b/.github/workflows/release-tag-version.yml index 2e7a9f5f54..da99fb072d 100644 --- a/.github/workflows/release-tag-version.yml +++ b/.github/workflows/release-tag-version.yml @@ -36,7 +36,7 @@ jobs: # xgo build - run: make release env: - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata - name: import gpg key id: import_gpg uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7.0.0 diff --git a/AGENTS.md b/AGENTS.md index 276023f234..16770e49fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ - Run `make fmt` to format `.go` files, and run `make lint-go` to lint them - Run `make lint-js` to lint `.ts` files - Run `make tidy` after any `go.mod` changes -- Run single go tests with `go test -tags 'sqlite sqlite_unlock_notify' -run '^TestName$' ./modulepath/` +- Run single go tests with `go test -run '^TestName$' ./modulepath/` - Run single js test files with `pnpm exec vitest ` - Run single playwright e2e test files with `GITEA_TEST_E2E_FLAGS='' make test-e2e` - Add the current year into the copyright header of new `.go` files diff --git a/Dockerfile b/Dockerfile index 323f06125f..b4f3e21c64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN make frontend FROM docker.io/library/golang:1.26-alpine3.23 AS build-env ARG GITEA_VERSION -ARG TAGS="sqlite sqlite_unlock_notify" +ARG TAGS="" ENV TAGS="bindata timetzdata $TAGS" ARG CGO_EXTRA_CFLAGS diff --git a/Dockerfile.rootless b/Dockerfile.rootless index 83c69cbd51..3edf85738a 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -12,7 +12,7 @@ RUN make frontend FROM docker.io/library/golang:1.26-alpine3.23 AS build-env ARG GITEA_VERSION -ARG TAGS="sqlite sqlite_unlock_notify" +ARG TAGS="" ENV TAGS="bindata timetzdata $TAGS" ARG CGO_EXTRA_CFLAGS diff --git a/Makefile b/Makefile index b6aa9371e5..a1f81738de 100644 --- a/Makefile +++ b/Makefile @@ -38,13 +38,10 @@ ifneq ($(findstring test-,$(MAKECMDGOALS)),) endif TAGS ?= -ifeq ($(GITEA_TEST_DATABASE),sqlite) - TAGS += sqlite sqlite_unlock_notify -endif TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags CGO_ENABLED ?= 0 -ifneq (,$(findstring sqlite,$(TAGS))$(findstring pam,$(TAGS))) +ifneq (,$(findstring sqlite_mattn,$(TAGS))$(findstring pam,$(TAGS))) CGO_ENABLED = 1 endif diff --git a/README.md b/README.md index 7ebeac97be..4c4fedd3b8 100644 --- a/README.md +++ b/README.md @@ -44,10 +44,6 @@ From the root of the source tree, run: TAGS="bindata" make build -or if SQLite support is required: - - TAGS="bindata sqlite sqlite_unlock_notify" make build - The `build` target is split into two sub-targets: - `make backend` which requires [Go Stable](https://go.dev/dl/), the required version is defined in [go.mod](/go.mod). diff --git a/README.zh-cn.md b/README.zh-cn.md index 8ccacc0fea..52c7781831 100644 --- a/README.zh-cn.md +++ b/README.zh-cn.md @@ -38,10 +38,6 @@ TAGS="bindata" make build -如果需要 SQLite 支持： - - TAGS="bindata sqlite sqlite_unlock_notify" make build - `build` 目标分为两个子目标： - `make backend` 需要 [Go Stable](https://go.dev/dl/)，所需版本在 [go.mod](/go.mod) 中定义。 diff --git a/README.zh-tw.md b/README.zh-tw.md index 4160fd0bd9..77ad6828db 100644 --- a/README.zh-tw.md +++ b/README.zh-tw.md @@ -38,10 +38,6 @@ TAGS="bindata" make build -如果需要 SQLite 支援： - - TAGS="bindata sqlite sqlite_unlock_notify" make build - `build` 目標分為兩個子目標： - `make backend` 需要 [Go Stable](https://go.dev/dl/)，所需版本在 [go.mod](/go.mod) 中定義。 diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 7bfd1fceeb..f6fcd2fbdf 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -1084,6 +1084,11 @@ "path": "github.com/redis/go-redis/v9/LICENSE", "licenseText": "Copyright (c) 2013 The github.com/redis/go-redis Authors.\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, + { + "name": "github.com/remyoudompheng/bigfft", + "path": "github.com/remyoudompheng/bigfft/LICENSE", + "licenseText": "Copyright (c) 2012 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "github.com/rhysd/actionlint", "path": "github.com/rhysd/actionlint/LICENSE.txt", @@ -1369,6 +1374,26 @@ "path": "gopkg.in/yaml.v3/LICENSE", "licenseText": "\nThis project is covered by two different licenses: MIT and Apache.\n\n#### MIT License ####\n\nThe following files were ported to Go from C files of libyaml, and thus\nare still covered by their original MIT license, with the additional\ncopyright staring in 2011 when the project was ported over:\n\n apic.go emitterc.go parserc.go readerc.go scannerc.go\n writerc.go yamlh.go yamlprivateh.go\n\nCopyright (c) 2006-2010 Kirill Simonov\nCopyright (c) 2006-2011 Kirill Simonov\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\nof the Software, and to permit persons to whom the Software is furnished to do\nso, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n### Apache License ###\n\nAll the remaining project files are covered by the Apache license:\n\nCopyright (c) 2011-2019 Canonical Ltd\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, + { + "name": "modernc.org/libc", + "path": "modernc.org/libc/LICENSE", + "licenseText": "Copyright (c) 2017 The Libc Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the names of the authors nor the names of the\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, + { + "name": "modernc.org/mathutil", + "path": "modernc.org/mathutil/LICENSE", + "licenseText": "Copyright (c) 2014 The mathutil Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the names of the authors nor the names of the\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, + { + "name": "modernc.org/memory", + "path": "modernc.org/memory/LICENSE", + "licenseText": "Copyright (c) 2017 The Memory Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the names of the authors nor the names of the\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, + { + "name": "modernc.org/sqlite", + "path": "modernc.org/sqlite/LICENSE", + "licenseText": "Copyright (c) 2017 The Sqlite Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\nlist of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\nthis list of conditions and the following disclaimer in the documentation\nand/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its contributors\nmay be used to endorse or promote products derived from this software without\nspecific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "mvdan.cc/xurls/v2", "path": "mvdan.cc/xurls/v2/LICENSE", diff --git a/contrib/ide/vscode/launch.json b/contrib/ide/vscode/launch.json index b80b826fc0..048428fc27 100644 --- a/contrib/ide/vscode/launch.json +++ b/contrib/ide/vscode/launch.json @@ -13,19 +13,6 @@ }, "args": ["web"], "showLog": true - }, - { - "name": "Launch (with SQLite3)", - "type": "go", - "request": "launch", - "mode": "debug", - "buildFlags": "-tags='sqlite sqlite_unlock_notify'", - "program": "${workspaceRoot}/main.go", - "env": { - "GITEA_WORK_DIR": "${workspaceRoot}", - }, - "args": ["web"], - "showLog": true } ] } diff --git a/contrib/ide/vscode/settings.json b/contrib/ide/vscode/settings.json index e33bccf902..d968e4daca 100644 --- a/contrib/ide/vscode/settings.json +++ b/contrib/ide/vscode/settings.json @@ -1,4 +1,4 @@ { - "go.buildTags": "'sqlite sqlite_unlock_notify'", + "go.buildTags": "", "go.testFlags": ["-v"] -} \ No newline at end of file +} diff --git a/contrib/ide/vscode/tasks.json b/contrib/ide/vscode/tasks.json index e35ae303b2..490e30338b 100644 --- a/contrib/ide/vscode/tasks.json +++ b/contrib/ide/vscode/tasks.json @@ -22,28 +22,6 @@ "args": ["build", "-o", "gitea.exe", "\"${workspaceRoot}\\main.go\""] }, "problemMatcher": ["$go"] - }, - { - "label": "Build (with SQLite3)", - "type": "shell", - "command": "go", - "group": "build", - "presentation": { - "echo": true, - "reveal": "always", - "focus": false, - "panel": "shared" - }, - "linux": { - "args": ["build", "-tags=\"sqlite sqlite_unlock_notify\"", "-o", "gitea", "${workspaceRoot}/main.go"] - }, - "osx": { - "args": ["build", "-tags=\"sqlite sqlite_unlock_notify\"", "-o", "gitea", "${workspaceRoot}/main.go"] - }, - "windows": { - "args": ["build", "-tags=\"sqlite sqlite_unlock_notify\"", "-o", "gitea.exe", "\"${workspaceRoot}\\main.go\""] - }, - "problemMatcher": ["$go"] } ] } diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index dd62cf8e83..2498050f5d 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -383,7 +383,7 @@ USER = root ;; ;DB_TYPE = sqlite3 ;PATH= ; defaults to data/gitea.db -;SQLITE_TIMEOUT = ; Query timeout defaults to: 500 +;SQLITE_TIMEOUT = ; Query timeout in milliseconds, defaults to: 20000 ;SQLITE_JOURNAL_MODE = ; defaults to sqlite database default (often DELETE), can be used to enable WAL mode. https://www.sqlite.org/pragma.html#pragma_journal_mode ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/flake.nix b/flake.nix index 7b9fbb193c..ca88b581a2 100644 --- a/flake.nix +++ b/flake.nix @@ -83,7 +83,7 @@ GO = "${go}/bin/go"; GOROOT = "${go}/share/go"; - TAGS = "sqlite sqlite_unlock_notify"; + TAGS = ""; STATIC = "true"; } // linuxOnlyEnv; diff --git a/go.mod b/go.mod index bcab996a71..bc88c0b79b 100644 --- a/go.mod +++ b/go.mod @@ -121,6 +121,7 @@ require ( google.golang.org/protobuf v1.36.11 gopkg.in/ini.v1 v1.67.2 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.50.0 mvdan.cc/xurls/v2 v2.6.0 strk.kbt.io/projects/go/libravatar v0.0.0-20260301104140-add494e31dab xorm.io/builder v0.3.13 @@ -238,6 +239,7 @@ require ( github.com/mrjones/oauth v0.0.0-20190623134757-126b35219450 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/nwaples/rardecode/v2 v2.2.2 // indirect github.com/oasdiff/yaml v0.0.9 // indirect github.com/oasdiff/yaml3 v0.0.12 // indirect @@ -255,6 +257,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rhysd/actionlint v1.7.12 // indirect github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -288,6 +291,9 @@ require ( golang.org/x/tools v0.44.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401020348-3a24fdc17823 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) ignore ( diff --git a/go.sum b/go.sum index 26c1df50ed..82849a19f6 100644 --- a/go.sum +++ b/go.sum @@ -563,8 +563,8 @@ github.com/msteinert/pam/v2 v2.1.0 h1:er5F9TKV5nGFuTt12ubtqPHEUdeBwReP7vd3wovidG github.com/msteinert/pam/v2 v2.1.0/go.mod h1:KT28NNIcDFf3PcBmNI2mIGO4zZJ+9RSs/At2PB3IDVc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niklasfasching/go-org v1.9.1 h1:/3s4uTPOF06pImGa2Yvlp24yKXZoTYM+nsIlMzfpg/0= github.com/niklasfasching/go-org v1.9.1/go.mod h1:ZAGFFkWvUQcpazmi/8nHqwvARpr1xpb+Es67oUGX/48= github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= @@ -952,24 +952,32 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.20.4 h1:J8+m2trkN+KKoE7jglyHYYYiaq5xmz2HoHJIiBlRzbE= -modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= +modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= +modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= +modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= +modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI= diff --git a/models/db/conn.go b/models/db/conn.go index de6f3cd5ec..944b6e7b97 100644 --- a/models/db/conn.go +++ b/models/db/conn.go @@ -10,6 +10,7 @@ import ( "net/url" "os" "path/filepath" + "slices" "strings" "code.gitea.io/gitea/modules/setting" @@ -31,7 +32,9 @@ type ConnOptions struct { } type SQLiteConnStrOptions struct { - FilePath string + FilePath string + // how long a concurrent query can wait for others (milliseconds), + // if timeout is reached, the error is something like "database is locked (SQLITE_BUSY)" BusyTimeout int JournalMode string } @@ -52,10 +55,21 @@ func GlobalConnOptions() ConnOptions { } } -const sqlDriverPostgresSchema = "postgresschema" +const ( + sqlDriverPostgresSchema = "postgresschema" + sqlDriverSQLite3 = "sqlite3" // although database type also has "sqlite3", they are different, for different purposes +) var makeSQLiteConnStr = func(opts SQLiteConnStrOptions) (string, string, error) { - return "", "", errors.New(`this Gitea binary was not built with SQLite3 support, get an official release or rebuild with: -tags sqlite,sqlite_unlock_notify`) + return "", "", errors.New(`this Gitea binary was not built with SQLite3 support, get an official release or rebuild with correct "-tags"`) +} + +func registerSQLiteConnStrMaker(fn func(opts SQLiteConnStrOptions) (string, string, error)) { + if slices.Contains(setting.SupportedDatabaseTypes, setting.DatabaseTypeSQLite3) { + panic("another sqlite3 driver has been registered") + } + setting.SupportedDatabaseTypes = append(setting.SupportedDatabaseTypes, setting.DatabaseTypeSQLite3) + makeSQLiteConnStr = fn } func ConnStrDefaultDatabase(opts ConnOptions) (string, string, error) { diff --git a/models/db/driver_sqlite_mattn.go b/models/db/driver_sqlite_mattn.go index 4988a43d3f..1fe2e35cba 100644 --- a/models/db/driver_sqlite_mattn.go +++ b/models/db/driver_sqlite_mattn.go @@ -1,4 +1,4 @@ -//go:build sqlite +//go:build sqlite_mattn && sqlite_unlock_notify // Copyright 2026 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT @@ -10,14 +10,11 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/modules/setting" - _ "github.com/mattn/go-sqlite3" ) func init() { - setting.SupportedDatabaseTypes = append(setting.SupportedDatabaseTypes, "sqlite3") - makeSQLiteConnStr = makeSQLiteConnStrMattnCGO + registerSQLiteConnStrMaker(makeSQLiteConnStrMattnCGO) } func makeSQLiteConnStrMattnCGO(opts SQLiteConnStrOptions) (string, string, error) { @@ -30,5 +27,5 @@ func makeSQLiteConnStrMattnCGO(opts SQLiteConnStrOptions) (string, string, error params = append(params, "_journal_mode="+opts.JournalMode) } connStr := fmt.Sprintf("file:%s?%s", opts.FilePath, strings.Join(params, "&")) - return "sqlite3", connStr, nil + return sqlDriverSQLite3, connStr, nil } diff --git a/models/db/driver_sqlite_modernc.go b/models/db/driver_sqlite_modernc.go new file mode 100644 index 0000000000..8202d5f6f4 --- /dev/null +++ b/models/db/driver_sqlite_modernc.go @@ -0,0 +1,41 @@ +//go:build !sqlite_mattn + +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// modernc driver is chosen as the default one (compared to mattn, ncruces) +// * mattn was used as default, but it requires CGO +// * the CI times are almost the same for these three (race detector must be disabled) +// * modernc increases the binary size about 2MB, ncruces increases about 7MB +// * compiling time: modernc is slightly slower than mattn, ncruces is the slowest + +package db + +import ( + "database/sql" + "fmt" + "strings" + + "modernc.org/sqlite" +) + +func init() { + // this driver contains huge amount of Golang code, so it is much slower when "-race" check is enabled. + registerSQLiteConnStrMaker(makeSQLiteConnStrModerncCCGO) + sql.Register(sqlDriverSQLite3, &sqlite.Driver{}) +} + +func makeSQLiteConnStrModerncCCGO(opts SQLiteConnStrOptions) (string, string, error) { + var params []string + // TODO: there is a changed behavior from mattn driver: + // * mattn driver can wait for pretty long time for concurrent accesses (not limited by the busy timeout) + // * but other drivers will report something like "database is locked (5) (SQLITE_BUSY)" if the timeout is reached + // Maybe we need to relax the busy timeout to a reasonable long time in the future + params = append(params, fmt.Sprintf("_pragma=busy_timeout(%d)", opts.BusyTimeout)) + params = append(params, "_txlock=immediate") + if opts.JournalMode != "" { + params = append(params, fmt.Sprintf("_pragma=journal_mode(%s)", opts.JournalMode)) + } + connStr := fmt.Sprintf("file:%s?%s", opts.FilePath, strings.Join(params, "&")) + return sqlDriverSQLite3, connStr, nil +} diff --git a/models/issues/issue_stats.go b/models/issues/issue_stats.go index adedaa3d3a..ddf6613870 100644 --- a/models/issues/issue_stats.go +++ b/models/issues/issue_stats.go @@ -35,12 +35,10 @@ const ( FilterModeYourRepositories ) -const ( - // MaxQueryParameters represents the max query parameters - // When queries are broken down in parts because of the number - // of parameters, attempt to break by this amount - MaxQueryParameters = 300 -) +// MaxQueryParameters represents the max query parameters +// When queries are broken down in parts because of the number +// of parameters, attempt to break by this amount +var MaxQueryParameters = 300 // CountIssuesByRepo map from repoID to number of issues matching the options func CountIssuesByRepo(ctx context.Context, opts *IssuesOptions) (map[int64]int64, error) { diff --git a/models/issues/issue_test.go b/models/issues/issue_test.go index b935c0fffd..99a09de0ec 100644 --- a/models/issues/issue_test.go +++ b/models/issues/issue_test.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" "xorm.io/builder" @@ -297,14 +298,11 @@ func TestIssue_ResolveMentions(t *testing.T) { func TestResourceIndex(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - var wg sync.WaitGroup - for i := range 100 { - wg.Add(1) - go func(i int) { + for i := range 25 { + wg.Go(func() { testInsertIssue(t, fmt.Sprintf("issue %d", i+1), "my issue", 0) - wg.Done() - }(i) + }) } wg.Wait() } @@ -317,18 +315,12 @@ func TestCorrectIssueStats(t *testing.T) { // maxQueryParameters + 10 issues into the testDatabase. // Each new issues will have a constant description "Bugs are nasty" // Which will be used later on. - + defer test.MockVariableValue(&issues_model.MaxQueryParameters, 25)() issueAmount := issues_model.MaxQueryParameters + 10 - var wg sync.WaitGroup for i := range issueAmount { - wg.Add(1) - go func(i int) { - testInsertIssue(t, fmt.Sprintf("Issue %d", i+1), "Bugs are nasty", 0) - wg.Done() - }(i) + testInsertIssue(t, fmt.Sprintf("Issue %d", i+1), "Bugs are nasty", 0) } - wg.Wait() // Now we will get all issueID's that match the "Bugs are nasty" query. issues, err := issues_model.Issues(t.Context(), &issues_model.IssuesOptions{ diff --git a/models/unittest/fixtures_test.go b/models/unittest/fixtures_test.go index 72944ec0db..d45ef33847 100644 --- a/models/unittest/fixtures_test.go +++ b/models/unittest/fixtures_test.go @@ -70,7 +70,7 @@ func prepareTestFixturesLoaders(t testing.TB) unittest.FixturesOptions { opts := unittest.FixturesOptions{Dir: filepath.Join(giteaRoot, "models", "fixtures"), Files: []string{ "user.yml", }} - require.NoError(t, unittest.CreateTestEngine(opts)) + require.NoError(t, unittest.CreateTestEngine(filepath.Join(t.TempDir(), "sqlite-test.db"), opts)) return opts } @@ -95,7 +95,7 @@ func TestFixturesLoader(t *testing.T) { func BenchmarkFixturesLoader(b *testing.B) { opts := prepareTestFixturesLoaders(b) - require.NoError(b, unittest.CreateTestEngine(opts)) + require.NoError(b, unittest.CreateTestEngine(filepath.Join(b.TempDir(), "sqlite-test.db"), opts)) loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts) require.NoError(b, err) loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts) diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 116fdab496..2a87a02fcd 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -56,7 +56,7 @@ func mainTest(m *testing.M, testOptsArg ...*TestOptions) int { giteaRoot := setting.GetGiteaTestSourceRoot() fixturesOpts := FixturesOptions{Dir: filepath.Join(giteaRoot, "models", "fixtures"), Files: testOpts.FixtureFiles} - if err := CreateTestEngine(fixturesOpts); err != nil { + if err := CreateTestEngine(filepath.Join(tempWorkPath, "sqlite-test.db"), fixturesOpts); err != nil { return testlogger.MainErrorf("Error creating test database engine: %v", err) } @@ -205,9 +205,9 @@ type FixturesOptions struct { Files []string } -// CreateTestEngine creates a memory database and loads the fixture data from fixturesDir -func CreateTestEngine(opts FixturesOptions) error { - driver, connStr, err := db.ConnStr(db.ConnOptions{Type: "sqlite3", SQLitePath: ":memory:"}) +// CreateTestEngine creates a test database and loads the fixture data from fixturesDir +func CreateTestEngine(testSQLiteFile string, opts FixturesOptions) error { + driver, connStr, err := db.ConnStr(db.ConnOptions{Type: setting.DatabaseTypeSQLite3, SQLitePath: testSQLiteFile, SQLiteBusyTimeout: 5000}) if err != nil { return err } diff --git a/modules/setting/database.go b/modules/setting/database.go index 2b069a6292..a65c8e9495 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -8,11 +8,13 @@ import ( "time" ) +const defaultSQLiteBusyTimeout = 20 * 1000 + var ( - // SupportedDatabaseTypes includes all XORM supported databases type, sqlite3 maybe added by `database_sqlite3.go` + // SupportedDatabaseTypes includes all XORM supported databases type, sqlite3 maybe added by the tag-controlled drivers SupportedDatabaseTypes = []string{"mysql", "postgres", "mssql"} // DatabaseTypeNames contains the friendly names for all database types - DatabaseTypeNames = map[string]string{"mysql": "MySQL", "postgres": "PostgreSQL", "mssql": "MSSQL", "sqlite3": "SQLite3"} + DatabaseTypeNames = map[string]string{"mysql": "MySQL", "postgres": "PostgreSQL", "mssql": "MSSQL", DatabaseTypeSQLite3: "SQLite3"} // Database holds the database settings Database = struct { @@ -39,7 +41,6 @@ var ( AutoMigration bool SlowQueryThreshold time.Duration }{ - SQLiteBusyTimeout: 500, IterateBufferSize: 50, } ) @@ -63,7 +64,13 @@ func loadDBSetting(rootCfg ConfigProvider) { Database.CharsetCollation = sec.Key("CHARSET_COLLATION").String() Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db")) - Database.SQLiteBusyTimeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) + + Database.SQLiteBusyTimeout = sec.Key("SQLITE_TIMEOUT").MustInt(defaultSQLiteBusyTimeout) + // mattn driver isn't really affected by this timeout, but other drivers are affected + // the default value was 500 (0.5s), to avoid breaking existing users, make sure the timeout is long enough (at least, 5 seconds) + if Database.SQLiteBusyTimeout < 5000 { + Database.SQLiteBusyTimeout = defaultSQLiteBusyTimeout + } Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("") Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2) @@ -85,8 +92,10 @@ func loadDBSetting(rootCfg ConfigProvider) { // DatabaseType FIXME: it is also used directly with "schemas.DBType", so the names must be consistent type DatabaseType string +const DatabaseTypeSQLite3 = "sqlite3" + func (t DatabaseType) IsSQLite3() bool { - return t == "sqlite3" + return t == DatabaseTypeSQLite3 } func (t DatabaseType) IsMySQL() bool { diff --git a/options/locale/locale_en-US.json b/options/locale/locale_en-US.json index 47fcac7ae7..c7ec133e57 100644 --- a/options/locale/locale_en-US.json +++ b/options/locale/locale_en-US.json @@ -313,7 +313,6 @@ "install.admin_email": "Email Address", "install.install_btn_confirm": "Install Gitea", "install.test_git_failed": "Could not test 'git' command: %v", - "install.sqlite3_not_available": "This Gitea version does not support SQLite3. Please download the official binary version from %s (not the 'gobuild' version).", "install.invalid_db_setting": "The database settings are invalid: %v", "install.invalid_db_table": "The database table \"%s\" is invalid: %v", "install.invalid_repo_path": "The repository root path is invalid: %v", diff --git a/routers/install/install.go b/routers/install/install.go index 718ede6564..3b21af6b03 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -123,7 +123,7 @@ func Install(ctx *context.Context) { func checkDatabase(ctx *context.Context, form *forms.InstallForm) bool { var err error - if (setting.Database.Type == "sqlite3") && + if (setting.Database.Type == setting.DatabaseTypeSQLite3) && len(setting.Database.Path) == 0 { ctx.Data["Err_DbPath"] = true ctx.RenderWithErrDeprecated(ctx.Tr("install.err_empty_db_path"), tplInstall, form) @@ -135,13 +135,8 @@ func checkDatabase(ctx *context.Context, form *forms.InstallForm) bool { defer db.UnsetDefaultEngine() if err = db.InitEngine(ctx); err != nil { - if strings.Contains(err.Error(), `Unknown database type: sqlite3`) { - ctx.Data["Err_DbType"] = true - ctx.RenderWithErrDeprecated(ctx.Tr("install.sqlite3_not_available", "https://docs.gitea.com/installation/install-from-binary"), tplInstall, form) - } else { - ctx.Data["Err_DbSetting"] = true - ctx.RenderWithErrDeprecated(ctx.Tr("install.invalid_db_setting", err), tplInstall, form) - } + ctx.Data["Err_DbSetting"] = true + ctx.RenderWithErrDeprecated(ctx.Tr("install.invalid_db_setting", err), tplInstall, form) return false } diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 9cad252e67..f78fc3e5f1 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -75,7 +75,7 @@ parts: set -x sed -i 's/os.Getuid()/1/g' modules/setting/setting.go npm install -g pnpm - TAGS="bindata sqlite sqlite_unlock_notify pam cert" make build + TAGS="bindata pam cert" make build install -D gitea "${SNAPCRAFT_PART_INSTALL}/gitea" cp -r options "${SNAPCRAFT_PART_INSTALL}/" diff --git a/tests/integration/api_issue_test.go b/tests/integration/api_issue_test.go index f8b6ab4dd9..e1708e90aa 100644 --- a/tests/integration/api_issue_test.go +++ b/tests/integration/api_issue_test.go @@ -149,16 +149,16 @@ func testAPICreateIssue(t *testing.T) { } func testAPICreateIssueParallel(t *testing.T) { - // FIXME: There seems to be a bug in github.com/mattn/go-sqlite3 with sqlite_unlock_notify, when doing concurrent writes to the same database, + // HINT: There seems to be a bug in github.com/mattn/go-sqlite3 with sqlite_unlock_notify, when doing concurrent writes to the same database, // some requests may get stuck in "go-sqlite3.(*SQLiteRows).Next", "go-sqlite3.(*SQLiteStmt).exec" and "go-sqlite3.unlock_notify_wait", - // because the "unlock_notify_wait" never returns and the internal lock never gets releases. + // because the "unlock_notify_wait" never returns and the internal lock never gets released. // // The trigger is: a previous test created issues and made the real issue indexer queue start processing, then this test does concurrent writing. // Adding this "Sleep" makes go-sqlite3 "finish" some internal operations before concurrent writes and then won't get stuck. // To reproduce: make a new test run these 2 tests enough times: // > func testBug() { for i := 0; i < 100; i++ { testAPICreateIssue(t); testAPICreateIssueParallel(t) } } // Usually the test gets stuck in fewer than 10 iterations without this "sleep". - time.Sleep(time.Second) + time.Sleep(100 * time.Millisecond) const body, title = "apiTestBody", "apiTestTitle"