Cleanup files

master
nexryai 4 weeks ago
parent 73dd262e48
commit 8c7675184a
Signed by: nexryai
GPG Key ID: 6CCD82A3AAFD1203
  1. 39
      .github/ISSUE_TEMPLATE/bug-report.md
  2. 5
      .github/ISSUE_TEMPLATE/config.yml
  3. 31
      .github/ISSUE_TEMPLATE/engine-request.md
  4. 21
      .github/ISSUE_TEMPLATE/feature-request.md
  5. 17
      .github/dependabot.yml
  6. 67
      .github/workflows/data-update.yml
  7. 192
      .github/workflows/integration.yml
  8. 28
      .github/workflows/security.yml
  9. 56
      .github/workflows/translations-update.yml
  10. 7
      .gitignore
  11. 104
      Makefile
  12. 10
      SECURITY.md
  13. 127
      utils/filtron.sh
  14. 1811
      utils/lib.sh
  15. 214
      utils/lib_go.sh
  16. 184
      utils/lib_nvm.sh
  17. 355
      utils/lib_redis.sh
  18. 129
      utils/lib_static.sh
  19. 68
      utils/lxc-searxng.env
  20. 578
      utils/lxc.sh
  21. 103
      utils/makefile.include
  22. 32
      utils/makefile.lxc
  23. 126
      utils/morty.sh
  24. 90
      utils/searx.sh
  25. 987
      utils/searxng.sh

@ -1,39 +0,0 @@
---
name: Bug report
about: Report a bug in SearXNG
title: ''
labels: bug
assignees: ''
---
<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SearXNG -->
**Version of SearXNG, commit number if you are using on master branch and stipulate if you forked SearXNG**
<!-- If you are running on master branch using git execute this command
in order to fetch the latest commit ID:
```
git log -1
```
If you are using searxng-docker then look at the bottom of the SearXNG page
and check for the version after "Powered by SearXNG"
Please also stipulate if you are using a forked version of SearXNG and
include a link to the fork source code.
-->
**How did you install SearXNG?**
<!-- Did you install SearXNG using the official wiki or using searxng-docker
or manually by executing the searx/webapp.py file? -->
**What happened?**
<!-- A clear and concise description of what the bug is. -->
**How To Reproduce**
<!-- How can we reproduce this issue? (as minimally and as precisely as possible) -->
**Expected behavior**
<!-- A clear and concise description of what you expected to happen. -->
**Screenshots & Logs**
<!-- If applicable, add screenshots, logs to help explain your problem. -->
**Additional context**
<!-- Add any other context about the problem here. -->

@ -1,5 +0,0 @@
blank_issues_enabled: true
contact_links:
- name: Questions & Answers (Q&A)
url: https://github.com/searxng/searxng/discussions/categories/q-a
about: Ask questions and find answers

@ -1,31 +0,0 @@
---
name: Engine request
about: Request a new engine in SearXNG
title: ''
labels: enhancement, engine request
assignees: ''
---
<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SearXNG -->
**Working URL to the engine**
<!-- Please check if the engine is responding correctly before submitting it. -->
**Why do you want to add this engine?**
<!-- What's special about this engine? Is it open source or libre? -->
**Features of this engine**
<!-- Features of this engine: Doesn't track its users, fast, easy to integrate, ... -->
**How can SearXNG fetch the information from this engine?**
<!-- List API URL, example code (using the correct markdown) and more
that could be useful for the developers in order to implement this engine.
If you don't know what to write, let this part blank.>
**Applicable category of this engine**
<!-- Where should this new engine fit in SearXNG? Current categories in SearXNG:
general, files, images, it, map, music, news, science, social media and videos.
You can add multiple categories at the same time. -->
**Additional context**
<!-- Add any other context about this engine here. -->

@ -1,21 +0,0 @@
---
name: Feature request
about: Request a new feature in SearXNG
title: ''
labels: enhancement
assignees: ''
---
<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SearXNG -->
**Is your feature request related to a problem? Please describe.**
<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
**Describe the solution you'd like**
<!-- A clear and concise description of what you want to happen. -->
**Describe alternatives you've considered**
<!-- A clear and concise description of any alternative solutions or features you've considered. -->
**Additional context**
<!-- Add any other context or screenshots about the feature request here. -->

@ -1,17 +0,0 @@
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "pip"
directory: "/"
schedule:
interval: "weekly"
day: "friday"
open-pull-requests-limit: 5
target-branch: "master"
- package-ecosystem: "npm"
directory: "/searx/static/themes/simple"
schedule:
interval: "weekly"
day: "friday"
open-pull-requests-limit: 5
target-branch: "master"

@ -1,67 +0,0 @@
name: "Update searx.data"
on:
schedule:
- cron: "59 23 28 * *"
workflow_dispatch:
jobs:
updateData:
name: Update data - ${{ matrix.fetch }}
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng'}}
strategy:
fail-fast: false
matrix:
fetch:
- update_ahmia_blacklist.py
- update_currencies.py
- update_external_bangs.py
- update_firefox_version.py
- update_languages.py
- update_wikidata_units.py
- update_engine_descriptions.py
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: |
sudo ./utils/searxng.sh install packages
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Install Python dependencies
run: |
make V=1 install
- name: Fetch data
env:
FETCH_SCRIPT: ./searxng_extra/update/${{ matrix.fetch }}
run: |
V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@v3
with:
commit-message: Update searx.data - ${{ matrix.fetch }}
committer: searxng-bot <noreply@github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
signoff: false
branch: update_data_${{ matrix.fetch }}
delete-branch: true
draft: false
title: 'Update searx.data - ${{ matrix.fetch }}'
body: |
Update searx.data - ${{ matrix.fetch }}
labels: |
data
- name: Check outputs
run: |
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

@ -1,192 +0,0 @@
name: Integration
on:
push:
branches: ["master"]
pull_request:
branches: ["master"]
jobs:
python:
name: Python ${{ matrix.python-version }}
runs-on: ubuntu-20.04
strategy:
matrix:
os: [ubuntu-20.04]
python-version: ["3.7", "3.8", "3.9", "3.10"]
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: |
sudo ./utils/searxng.sh install packages
sudo apt install firefox
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: Install Python dependencies
if: steps.cache-python.outputs.cache-hit != 'true'
run: |
make V=1 install
make V=1 gecko.driver
- name: Run tests
run: make V=1 ci.test
- name: Test coverage
run: make V=1 test.coverage
- name: Store coverage result
uses: actions/upload-artifact@v2
with:
name: coverage-${{ matrix.python-version }}
path: coverage/
retention-days: 60
themes:
name: Themes
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: Install node dependencies
run: make V=1 node.env
- name: Build themes
run: make V=1 themes.all
documentation:
name: Documentation
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: '0'
persist-credentials: false
- name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: Build documentation
run: |
make V=1 docs.clean docs.html
- name: Deploy
if: github.ref == 'refs/heads/master'
uses: JamesIves/github-pages-deploy-action@3.7.1
with:
GITHUB_TOKEN: ${{ github.token }}
BRANCH: gh-pages
FOLDER: dist/docs
CLEAN: true # Automatically remove deleted files from the deploy branch
SINGLE_COMMIT: True
COMMIT_MESSAGE: build from commit ${{ github.sha }}
babel:
name: Update translations branch
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
needs:
- python
- themes
- documentation
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: weblate & git setup
env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: |
mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot"
- name: Update transations
id: update
run: |
git restore utils/brand.env
make V=1 weblate.push.translations
dockers:
name: Docker
if: github.ref == 'refs/heads/master'
needs:
- python
- themes
- documentation
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
runs-on: ubuntu-20.04
steps:
- name: Checkout
if: env.DOCKERHUB_USERNAME != null
uses: actions/checkout@v2
with:
# make sure "make docker.push" can get the git history
fetch-depth: '0'
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: Set up QEMU
if: env.DOCKERHUB_USERNAME != null
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
if: env.DOCKERHUB_USERNAME != null
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
if: env.DOCKERHUB_USERNAME != null
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
if: env.DOCKERHUB_USERNAME != null
run: make -e GIT_URL=$(git remote get-url origin) docker.buildx

@ -1,28 +0,0 @@
name: "Security checks"
on:
schedule:
- cron: "42 05 * * *"
workflow_dispatch:
jobs:
dockers:
name: Trivy ${{ matrix.image }}
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:
image-ref: 'searxng/searxng:latest'
ignore-unfixed: false
vuln-type: 'os,library'
severity: 'UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL'
format: 'sarif'
output: 'trivy-results.sarif'
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: 'trivy-results.sarif'

@ -1,56 +0,0 @@
name: "Update translations"
on:
schedule:
- cron: "05 07 * * 5"
workflow_dispatch:
jobs:
babel:
name: "create PR for additons from weblate"
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v2
with:
path: ./local
key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
- name: weblate & git setup
env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
run: |
mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot"
- name: Merge and push transation updates
run: |
make V=1 weblate.translations.commit
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@v3
with:
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
commit-message: Update translations
committer: searxng-bot <searxng-bot@users.noreply.github.com>
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
signoff: false
branch: translations_update
delete-branch: true
draft: false
title: 'Update translations'
body: |
Update translations
labels: |
translation

7
.gitignore vendored

@ -19,3 +19,10 @@ gh-pages/
/package-lock.json
/node_modules/
utils/*.sh
utils/lxc-searxng.env
utils/makefile*
Makefile
SECURITY.md
.github

@ -1,104 +0,0 @@
# -*- coding: utf-8; mode: makefile-gmake -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
.DEFAULT_GOAL=help
export MTOOLS=./manage
include utils/makefile.include
all: clean install
PHONY += help
help:
@./manage --help
@echo '----'
@echo 'run - run developer instance'
@echo 'install - developer install of SearxNG into virtualenv'
@echo 'uninstall - uninstall developer installation'
@echo 'clean - clean up working tree'
@echo 'search.checker - check search engines'
@echo 'test - run shell & CI tests'
@echo 'test.shell - test shell scripts'
@echo 'ci.test - run CI tests'
PHONY += run
run: install
$(Q)./manage webapp.run
PHONY += install uninstall
install uninstall:
$(Q)./manage pyenv.$@
PHONY += clean
clean: py.clean docs.clean node.clean nvm.clean test.clean
$(Q)./manage build_msg CLEAN "common files"
$(Q)find . -name '*.orig' -exec rm -f {} +
$(Q)find . -name '*.rej' -exec rm -f {} +
$(Q)find . -name '*~' -exec rm -f {} +
$(Q)find . -name '*.bak' -exec rm -f {} +
lxc.clean:
$(Q)rm -rf lxc-env
PHONY += search.checker search.checker.%
search.checker: install
$(Q)./manage pyenv.cmd searx-checker -v
search.checker.%: install
$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
PHONY += test ci.test test.shell
ci.test: test.yamllint test.black test.pyright test.pylint test.unit test.robot test.rst test.pybabel
test: test.yamllint test.black test.pyright test.pylint test.unit test.robot test.rst test.shell
test.shell:
$(Q)shellcheck -x -s dash \
dockerfiles/docker-entrypoint.sh
$(Q)shellcheck -x -s bash \
utils/brand.env \
$(MTOOLS) \
utils/lib.sh \
utils/lib_nvm.sh \
utils/lib_static.sh \
utils/lib_go.sh \
utils/lib_redis.sh \
utils/filtron.sh \
utils/searx.sh \
utils/searxng.sh \
utils/morty.sh \
utils/lxc.sh \
utils/lxc-searxng.env
$(Q)$(MTOOLS) build_msg TEST "$@ OK"
# wrap ./manage script
MANAGE += buildenv
MANAGE += weblate.translations.commit weblate.push.translations
MANAGE += data.all data.languages data.useragents data.osm_keys_tags
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += docker.build docker.push docker.buildx
MANAGE += gecko.driver
MANAGE += node.env node.clean
MANAGE += py.build py.clean
MANAGE += pyenv pyenv.install pyenv.uninstall
MANAGE += pypi.upload pypi.upload.test
MANAGE += format.python
MANAGE += test.yamllint test.pylint test.pyright test.black test.pybabel test.unit test.coverage test.robot test.rst test.clean
MANAGE += themes.all themes.simple themes.simple.test pygments.less
MANAGE += static.build.commit static.build.drop static.build.restore
MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
PHONY += $(MANAGE)
$(MANAGE):
$(Q)$(MTOOLS) $@
# short hands of selected targets
PHONY += docs docker themes
docs: docs.html
docker: docker.build
themes: themes.all

@ -1,10 +0,0 @@
# Security Policy
We love responsible reports of (potential) security issues in SearXNG.
You can contact us at security@searxng.org.
Be sure to provide as much information as possible and if found
also reproduction steps of the identified vulnerability. Also
add the specific URL of the project as well as code you found
the issue in to your report.

@ -1,127 +0,0 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2001
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
PUBLIC_URL="${PUBLIC_URL:-${SEARXNG_URL}}"
FILTRON_ETC="/etc/filtron"
SERVICE_NAME="filtron"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
APACHE_FILTRON_SITE="searx.conf"
NGINX_FILTRON_SITE="searx.conf"
# ----------------------------------------------------------------------------
usage() {
# ----------------------------------------------------------------------------
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") remove all
$(basename "$0") apache remove
$(basename "$0") nginx remove
remove all : drop all components of the filtron service
apache remove : drop apache site ${APACHE_FILTRON_SITE}
nginx remove : drop nginx site ${NGINX_FILTRON_SITE}
environment:
PUBLIC_URL : ${PUBLIC_URL}
EOF
[[ -n ${1} ]] && err_msg "$1"
}
main() {
local _usage="unknown or missing $1 command $2"
case $1 in
-h|--help) usage; exit 0;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
*) usage "unknown or missing command $1"; exit 42;;
esac
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
rst_para "\
It goes without saying that this script can only be used to remove
installations that were installed with this script."
if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
return 42
fi
drop_service_account "${SERVICE_USER}"
rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
if service_is_available "${PUBLIC_URL}"; then
MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10
fi
}
remove_apache_site() {
rst_title "Remove Apache site $APACHE_FILTRON_SITE"
rst_para "\
This removes apache site ${APACHE_FILTRON_SITE}."
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
apache_remove_site "$APACHE_FILTRON_SITE"
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_FILTRON_SITE"
rst_para "\
This removes nginx site ${NGINX_FILTRON_SITE}."
! nginx_is_installed && err_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
fi
nginx_remove_app "$FILTRON_FILTRON_SITE"
}
# ----------------------------------------------------------------------------
main "$@"
# ----------------------------------------------------------------------------

File diff suppressed because it is too large Load Diff

@ -1,214 +0,0 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# Tools to install and maintain golang [1] binaries & packages.
#
# [1] https://golang.org/doc/devel/release#policy
#
# A simple *helloworld* test with user 'my_user' :
#
# sudo -H adduser my_user
# ./manage go.golang go1.17.3 my_user
# ./manage go.install github.com/go-training/helloworld@latest my_user
# ./manage go.bash my_user
# $ helloword
# Hello World!!
#
# Don't forget to remove 'my_user': sudo -H deluser --remove-home my_user
# shellcheck source=utils/lib.sh
. /dev/null
# shellcheck disable=SC2034
declare main_cmd
# configure golang environment
# ----------------------------
[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3"
GO_DL_URL="https://golang.org/dl"
# implement go functions
# -----------------------
go.help(){
cat <<EOF
go.:
ls : list golang binary archives (stable)
golang : (re-) install golang binary in user's \$HOME/local folder
install : install go package in user's \$HOME/go-apps folder
bash : start bash interpreter with golang environment sourced
EOF
}
go.ls(){
python3 <<EOF
import sys, json, requests
resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
for ver in json.loads(resp.text):
if not ver['stable']:
continue
for f in ver['files']:
if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
continue
print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
EOF
}
go.ver_info(){
# print informations about a golang distribution. To print filename
# sha256 and size of the archive that fits to your OS and host:
#
# go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
#
# usage: go.ver_info <go-vers> <kind> <os> <arch> [filename|sha256|size]
#
# kind: [archive|source|installer]
# os: [darwin|freebsd|linux|windows]
# arch: [amd64|arm64|386|armv6l|ppc64le|s390x]
python3 - "$@" <<EOF
import sys, json, requests
resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
for ver in json.loads(resp.text):
if ver['version'] != sys.argv[1]:
continue
for f in ver['files']:
if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
continue
for x in sys.argv[5:]:
print(f[x])
sys.exit(0)
sys.exit(42)
EOF
}
go.os() {
local OS
case "$(command uname -a)xx" in
Linux\ *) OS=linux ;;
Darwin\ *) OS=darwin ;;
FreeBSD\ *) OS=freebsd ;;
CYGWIN* | MSYS* | MINGW*) OS=windows ;;
*) die 42 "OS is unknown: $(command uname -a)" ;;
esac
echo "${OS}"
}
go.arch() {
local ARCH
case "$(command uname -m)" in
"x86_64") ARCH=amd64 ;;
"aarch64") ARCH=arm64 ;;
"armv6" | "armv7l") ARCH=armv6l ;;
"armv8") ARCH=arm64 ;;
.*386.*) ARCH=386 ;;
ppc64*) ARCH=ppc64le ;;
*) die 42 "ARCH is unknown: $(command uname -m)" ;;
esac
echo "${ARCH}"
}
go.golang() {
# install golang binary in user's $HOME/local folder:
#
# go.golang ${GO_VERSION} ${SERVICE_USER}
#
# usage: go.golang <go-vers> [<username>]
local version fname sha size user userpr
local buf=()
version="${1:-${GO_VERSION}}"
user="${2:-${USERNAME}}"
userpr=" ${_Yellow}|${user}|${_creset} "
rst_title "Install Go in ${user}'s HOME" section
mapfile -t buf < <(
go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
)
if [ ${#buf[@]} -eq 0 ]; then
die 42 "can't find info of golang version: ${version}"
fi
fname="${buf[0]}"
sha="${buf[1]}"
size="$(numfmt --to=iec "${buf[2]}")"
info_msg "Download go binary ${fname} (${size}B)"
cache_download "${GO_DL_URL}/${fname}" "${fname}"
pushd "${CACHE}" &> /dev/null
echo "${sha} ${fname}" > "${fname}.sha256"
if ! sha256sum -c "${fname}.sha256" >/dev/null; then
die 42 "downloaded file ${fname} checksum does not match"
else
info_msg "${fname} checksum OK"
fi
popd &> /dev/null
info_msg "install golang"
tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
mkdir -p \$HOME/local
rm -rf \$HOME/local/go
tar -C \$HOME/local -xzf ${CACHE}/${fname}
echo "export GOPATH=\$HOME/go-apps" > \$HOME/.go_env
echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env
EOF
info_msg "test golang installation"
sudo -i -u "${user}" <<EOF
source \$HOME/.go_env
command -v go
go version
EOF
}
go.install() {
# install go package in user's $HOME/go-apps folder:
#
# go.install github.com/go-training/helloworld@lates ${SERVICE_USER}
#
# usage: go.install <package> [<username>]
local package user userpr
package="${1}"
user="${2:-${USERNAME}}"
userpr=" ${_Yellow}|${user}|${_creset} "
if [ -z "${package}" ]; then
die 42 "${FUNCNAME[0]}() - missing argument: <package>"
fi
tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
source \$HOME/.go_env
go install -v ${package}
EOF
}
go.bash() {
# start bash interpreter with golang environment sourced
#
# go.bash ${SERVICE_USER}
#
# usage: go.bash [<username>]
local user
user="${1:-${USERNAME}}"
sudo -i -u "${user}" bash --init-file "~${user}/.go_env"
}
go.version(){
local user
user="${1:-${USERNAME}}"
sudo -i -u "${user}" <<EOF
source \$HOME/.go_env
go version | cut -d' ' -f 3
EOF
}

@ -1,184 +0,0 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# Tools to install and maintain NVM versions manager for Node.js
#
# [1] https://github.com/nvm-sh/nvm
# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285
# shellcheck source=utils/lib.sh
. /dev/null
declare main_cmd
# configure nvm environment
# -------------------------
NVM_LOCAL_FOLDER=.nvm
[[ -z "${NVM_GIT_URL}" ]] && NVM_GIT_URL="https://github.com/nvm-sh/nvm.git"
[[ -z "${NVM_MIN_NODE_VER}" ]] && NVM_MIN_NODE_VER="16.13.0"
# initalize nvm environment
# -------------------------
nvm.env() {
source "${NVM_DIR}/nvm.sh"
source "${NVM_DIR}/bash_completion"
[ "$VERBOSE" = "1" ] && info_msg "sourced NVM environment from ${NVM_DIR}"
}
nvm.is_installed() {
# is true if NVM is installed / in $HOME or even in <repo-root>/.nvm
[[ -f "${NVM_DIR}/nvm.sh" ]]
}
if [[ -z "${NVM_DIR}" ]]; then
# nvm is not pre-intalled in $HOME. Prepare for using nvm from <repo-root>
NVM_DIR="$(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}"
fi
export NVM_DIR
if nvm.is_installed; then
nvm.env
else
# if nvm is not installed, use this function as a wrapper
nvm() {
nvm.ensure
nvm "$@"
}
fi
# implement nvm functions
# -----------------------
nvm.is_local() {
# is true if NVM is installed in <repo-root>/.nvm
[ "${NVM_DIR}" = "$(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}" ]
}
nvm.min_node() {
# usage: nvm.min_node 16.3.0
#
# Is true if minimal Node.js version is installed.
local min_v
local node_v
local higher_v
if ! command -v node >/dev/null; then
warn_msg "Node.js is not yet installed"
return 42
fi
min_v="${1}"
node_v="$(node --version)"
node_v="${node_v:1}" # remove 'v' from 'v16.3.0'
if ! [ "${min_v}" = "${node_v}" ]; then
higher_v="$(echo -e "$min_v\n${node_v}" | sort -Vr | head -1)"
if [ "${min_v}" = "${higher_v}" ]; then
return 42
fi
fi
}
# implement nvm command line
# --------------------------
nvm.help() {
cat <<EOF
nvm.: use nvm (without dot) to execute nvm commands directly
install : install NVM locally at $(git rev-parse --show-toplevel)/${NVM_LOCAL_FOLDER}
clean : remove NVM installation
status : prompt some status informations about nvm & node
nodejs : install Node.js latest LTS
cmd ... : run command ... in NVM environment
bash : start bash interpreter with NVM environment sourced
EOF
}
nvm.install() {
local NVM_VERSION_TAG
info_msg "install (update) NVM at ${NVM_DIR}"
if [[ -d "${NVM_DIR}" ]] ; then
info_msg "already cloned at: ${NVM_DIR}"
pushd "${NVM_DIR}" &> /dev/null
git fetch --all | prefix_stdout " ${_Yellow}||${_creset} "
else
info_msg "clone: ${NVM_GIT_URL}"
git clone "${NVM_GIT_URL}" "${NVM_DIR}" 2>&1 | prefix_stdout " ${_Yellow}||${_creset} "
pushd "${NVM_DIR}" &> /dev/null
git config --local advice.detachedHead false
fi
NVM_VERSION_TAG="$(git rev-list --tags --max-count=1)"
NVM_VERSION_TAG="$(git describe --abbrev=0 --tags --match "v[0-9]*" "${NVM_VERSION_TAG}")"
info_msg "checkout ${NVM_VERSION_TAG}"
git checkout "${NVM_VERSION_TAG}" 2>&1 | prefix_stdout " ${_Yellow}||${_creset} "
popd &> /dev/null
if [ -f "${REPO_ROOT}/.nvm_packages" ]; then
cp "${REPO_ROOT}/.nvm_packages" "${NVM_DIR}/default-packages"
fi
nvm.env
}
nvm.clean() {
if ! nvm.is_installed; then
build_msg CLEAN "[NVM] not installed"
return
fi
if ! nvm.is_local; then
build_msg CLEAN "[NVM] can't remove nvm from ${NVM_DIR}"
return
fi
if [ -n "${NVM_DIR}" ]; then
build_msg CLEAN "[NVM] drop $(realpath --relative-to=. "${NVM_DIR}")/"
rm -rf "${NVM_DIR}"
fi
}
nvm.status() {
if command -v node >/dev/null; then
info_msg "Node.js is installed at $(command -v node)"
info_msg "Node.js is version $(node --version)"
if ! nvm.min_node "${NVM_MIN_NODE_VER}"; then
warn_msg "minimal Node.js version is ${NVM_MIN_NODE_VER}"
fi
else
warn_msg "Node.js is mot installed"
fi
if command -v npm >/dev/null; then
info_msg "npm is installed at $(command -v npm)"
info_msg "npm is version $(npm --version)"
else
warn_msg "npm is not installed"
fi
if nvm.is_installed; then
info_msg "NVM is installed at ${NVM_DIR}"
else
warn_msg "NVM is not installed"
info_msg "to install NVM and Node.js (LTS) use: ${main_cmd} nvm.nodejs"
fi
}
nvm.nodejs() {
nvm install
nvm.status
}
nvm.bash() {
nvm.ensure
bash --init-file <(cat "${NVM_DIR}/nvm.sh" "${NVM_DIR}/bash_completion")
}
nvm.cmd() {
nvm.ensure
"$@"
}
nvm.ensure() {
if ! nvm.is_installed; then
nvm.install
fi
}

@ -1,355 +0,0 @@
#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# Tools to build and install redis [1] binaries & packages.
#
# [1] https://redis.io/download#installation
#
# 1. redis.devpkg (sudo)
# 2. redis.build
# 3. redis.install (sudo)
#
# systemd commands::
#
# sudo -H systemctl status searxng-redis
# sudo -H journalctl -u searxng-redis
# sudo -H journalctl --vacuum-size=1M
#
# Test socket connection from client (local user)::
#
# $ sudo -H ./manage redis.addgrp "${USER}"
# # logout & login to get member of group
# $ groups
# ... searxng-redis ...
# $ source /usr/local/searxng-redis/.redis_env
# $ which redis-cli
# /usr/local/searxng-redis/.local/bin/redis-cli
#
# $ redis-cli -s /usr/local/searxng-redis/redis.sock
# redis /usr/local/searxng-redis/redis.sock> set foo bar
# OK
# redis /usr/local/searxng-redis/redis.sock> get foo
# "bar"
# [CTRL-D]
# shellcheck disable=SC2091
# shellcheck source=utils/lib.sh
. /dev/null
REDIS_GIT_URL="https://github.com/redis/redis.git"
REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}"
REDIS_USER="searxng-redis"
REDIS_GROUP="searxng-redis"
REDIS_HOME="/usr/local/${REDIS_USER}"
REDIS_HOME_BIN="${REDIS_HOME}/.local/bin"
REDIS_ENV="${REDIS_HOME}/.redis_env"
REDIS_SERVICE_NAME="searxng-redis"
REDIS_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${REDIS_SERVICE_NAME}.service"
# binaries to compile & install
REDIS_INSTALL_EXE=(redis-server redis-benchmark redis-cli)
# link names of redis-server binary
REDIS_LINK_EXE=(redis-sentinel redis-check-rdb redis-check-aof)
REDIS_CONF="${REDIS_HOME}/redis.conf"
REDIS_CONF_TEMPLATE=$(cat <<EOF
# Note that in order to read the configuration file, Redis must be
# started with the file path as first argument:
#
# ./redis-server /path/to/redis.conf
# bind 127.0.0.1 -::1
protected-mode yes
# Accept connections on the specified port, default is 6379 (IANA #815344).
# If port 0 is specified Redis will not listen on a TCP socket.
port 0
# Specify the path for the Unix socket that will be used to listen for
# incoming connections.
unixsocket ${REDIS_HOME}/run/redis.sock
unixsocketperm 770
# The working directory.
dir ${REDIS_HOME}/run
# If you run Redis from upstart or systemd, Redis can interact with your
# supervision tree.
supervised auto
pidfile ${REDIS_HOME}/run/redis.pid
# log to the system logger
syslog-enabled yes
EOF
)
redis.help(){
cat <<EOF
redis.:
devpkg : install essential packages to compile redis
build : build redis binaries at $(redis._get_dist)
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
remove : delete user (${REDIS_USER}) and remove service (${REDIS_SERVICE_NAME})
shell : start bash interpreter from user ${REDIS_USER}
src : clone redis source code to <path> and checkput ${REDIS_GIT_TAG}
useradd : create user (${REDIS_USER}) at ${REDIS_HOME}
userdel : delete user (${REDIS_USER})
addgrp : add <user> to group (${REDIS_USER})
rmgrp : remove <user> from group (${REDIS_USER})
EOF
}
redis.devpkg() {
# Uses OS package manager to install the essential packages to build and
# compile sources
sudo_or_exit
case ${DIST_ID} in
ubuntu|debian)
pkg_install git build-essential gawk
;;
arch)
pkg_install git base-devel
;;
fedora)
pkg_install git @development-tools
;;
centos)
pkg_install git
yum groupinstall "Development Tools" -y
;;
*)
err_msg "$DIST_ID-$DIST_VERS: No rules to install development tools from OS."
return 42
;;
esac
}
redis.build() {
# usage: redis.build
rst_title "get redis sources" section
redis.src "${CACHE}/redis"
if ! required_commands gcc nm make gawk ; then
info_msg "install development tools to get missing command(s) .."
if [[ -n ${SUDO_USER} ]]; then
sudo -H "$0" redis.devpkg
else
redis.devpkg
fi
fi
rst_title "compile redis sources" section
pushd "${CACHE}/redis" &>/dev/null
if ask_yn "Do you run 'make distclean' first'?" Yn; then
$(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout
fi
$(bash.cmd) -c "make" 2>&1 | prefix_stdout
if ask_yn "Do you run 'make test'?" Ny; then
$(bash.cmd) -c "make test" | prefix_stdout
fi
popd &>/dev/null
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
mkdir -p "$(redis._get_dist)"
cd "${CACHE}/redis/src"
cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)"
EOF
info_msg "redis binaries available at $(redis._get_dist)"
}
redis.install() {
sudo_or_exit
(
set -e
redis.useradd
redis._install_bin
redis._install_conf
redis._install_service
)
dump_return $?
}
redis.remove() {
sudo_or_exit
(
set -e
redis._remove_service
redis.userdel
)
dump_return $?
}
redis.shell() {
interactive_shell "${REDIS_USER}"
}
redis.src() {
# usage: redis.src "${CACHE}/redis"
local dest="${1:-${CACHE}/redis}"
if [ -d "${dest}" ] ; then
info_msg "already cloned: $dest"
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
cd "${dest}"
git fetch --all
git reset --hard tags/${REDIS_GIT_TAG}
EOF
else
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
mkdir -p "$(dirname "$dest")"
cd "$(dirname "$dest")"
git clone "${REDIS_GIT_URL}" "${dest}"
EOF
tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout
cd "${dest}"
git checkout tags/${REDIS_GIT_TAG} -b "build-branch"
EOF
fi
}
redis.useradd(){
# usage: redis.useradd
rst_title "add user ${REDIS_USER}" section
echo
sudo_or_exit
# create user account
tee_stderr 0.5 <<EOF | sudo -H bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "${REDIS_HOME}" \
--comment 'user that runs a redis instance' "${REDIS_USER}"
mkdir -p "${REDIS_HOME}"
chown -R "${REDIS_USER}:${REDIS_GROUP}" "${REDIS_HOME}"
groups "${REDIS_USER}"
EOF
# create App-ENV and add source it in the .profile
tee_stderr 0.5 <<EOF | sudo -H -u "${REDIS_USER}" bash | prefix_stdout
mkdir -p "${REDIS_HOME_BIN}"
echo "export PATH=${REDIS_HOME_BIN}:\\\$PATH" > "${REDIS_ENV}"
grep -qFs -- 'source "${REDIS_ENV}"' ~/.profile || echo 'source "${REDIS_ENV}"' >> ~/.profile
EOF
}
redis.userdel() {
sudo_or_exit
drop_service_account "${REDIS_USER}"
groupdel "${REDIS_GROUP}" 2>&1 | prefix_stdout || true
}
redis.addgrp() {
# usage: redis.addgrp <user>
[[ -z $1 ]] && die_caller 42 "missing argument <user>"
sudo -H gpasswd -a "$1" "${REDIS_GROUP}"
}
redis.rmgrp() {
# usage: redis.rmgrp <user>
[[ -z $1 ]] && die_caller 42 "missing argument <user>"
sudo -H gpasswd -d "$1" "${REDIS_GROUP}"
}
# private redis. functions
# ------------------------
redis._install_bin() {
local src
src="$(redis._get_dist)"
(
set -e
for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do
install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \
"${src}/${redis_exe}" "${REDIS_HOME_BIN}"
done
pushd "${REDIS_HOME_BIN}" &> /dev/null
for redis_exe in "${REDIS_LINK_EXE[@]}"; do
info_msg "link redis-server --> ${redis_exe}"
sudo -H -u "${REDIS_USER}" ln -sf redis-server "${redis_exe}"
done
popd &> /dev/null
)
}
redis._install_conf() {
sudo -H -u "${REDIS_USER}" bash <<EOF
mkdir -p "${REDIS_HOME}/run"
echo '${REDIS_CONF_TEMPLATE}' > "${REDIS_CONF}"
EOF
}
redis._install_service() {
systemd_install_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
}
redis._remove_service() {
systemd_remove_service "${REDIS_SERVICE_NAME}" "${REDIS_SYSTEMD_UNIT}"
}
redis._get_dist() {
if [ -z "${REDIS_DIST}" ]; then
echo "${REPO_ROOT}/dist/redis/${REDIS_GIT_TAG}/$(redis._arch)"
else
echo "${REDIS_DIST}"
fi
}
redis._arch() {
local ARCH
case "$(command uname -m)" in
"x86_64") ARCH=amd64 ;;
"aarch64") ARCH=arm64 ;;
"armv6" | "armv7l") ARCH=armv6l ;;
"armv8") ARCH=arm64 ;;
.*386.*) ARCH=386 ;;
ppc64*) ARCH=ppc64le ;;
*) die 42 "ARCH is unknown: $(command uname -m)" ;;
esac
echo "${ARCH}"
}
# TODO: move this to the right place ..
bash.cmd(){
# print cmd to get a bash in a non-root mode, even if we are in a sudo
# context.
local user="${USER}"
local bash_cmd="bash"
if [ -n "${SUDO_USER}" ] && [ "root" != "${SUDO_USER}" ] ; then
user="${SUDO_USER}"
bash_cmd="sudo -H -u ${SUDO_USER} bash"
fi