Browse Source

[clean up] drop obsolete searx, filtron and morty install scripts

Since ./utils/searxng.sh is implemented, the old installation procedures from
filtron, morty and searx can be removed.

For users who want to upgrade, the procedures for removing old installations
have still been retained.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
master^2
Markus Heiser 2 months ago
parent
commit
692708aa77
  1. 52
      .config.sh
  2. 2
      .github/workflows/data-update.yml
  3. 6
      .github/workflows/integration.yml
  4. 4
      Makefile
  5. 2
      manage
  6. 527
      utils/filtron.sh
  7. 208
      utils/lib_install.sh
  8. 1
      utils/lxc.sh
  9. 457
      utils/morty.sh
  10. 1033
      utils/searx.sh
  11. 129
      utils/templates/etc/filtron/rules.json
  12. 28
      utils/templates/etc/httpd/sites-available/morty.conf
  13. 33
      utils/templates/etc/httpd/sites-available/searxng.conf:filtron
  14. 34
      utils/templates/etc/httpd/sites-available/searxng.conf:uwsgi
  15. 11
      utils/templates/etc/nginx/default.apps-available/morty.conf
  16. 16
      utils/templates/etc/nginx/default.apps-available/searxng.conf:filtron

52
.config.sh

@ -1,52 +0,0 @@
# -*- coding: utf-8; mode: sh -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck shell=bash disable=SC2034
#
# This file should be edited only ones just before the installation of any
# service is done. After the installation of the searx service a copy of this
# file is placed into the $SEARXNG_SRC of the instance, e.g.::
#
# /usr/local/searx/searx-src/.config.sh
#
# .. hint::
#
# Before you change a value here, You have to fully uninstall any previous
# installation of searx, morty and filtron services!
# utils/searx.sh
# --------------
# The setup of the SearXNG instance is done in the settings.yml
# (SEARXNG_SETTINGS_PATH). Read the remarks in [1] carefully and don't forget to
# rebuild instance's environment (make buildenv) if needed. The settings.yml
# file of an already installed instance is shown by::
#
# $ ./utils/searx.sh --help
# ---- SearXNG instance setup (already installed)
# SEARXNG_SETTINGS_PATH : /etc/searxng/settings.yml
# SEARXNG_SRC : /usr/local/searx/searx-src
#
# [1] https://docs.searxng.org/admin/engines/settings.html
# utils/filtron.sh
# ----------------
# FILTRON_API="127.0.0.1:4005"
# FILTRON_LISTEN="127.0.0.1:4004"
# utils/morty.sh
# --------------
# morty listen address
# MORTY_LISTEN="127.0.0.1:3000"
# PUBLIC_URL_PATH_MORTY="/morty/"
# system services
# ---------------
# Common $HOME folder of the service accounts
# SERVICE_HOME_BASE="/usr/local"
# **experimental**: Set SERVICE_USER to run all services by one account, but be
# aware that removing discrete components might conflict!
# SERVICE_USER=searx

2
.github/workflows/data-update.yml

@ -26,7 +26,7 @@ jobs:
- name: Install Ubuntu packages
run: |
sudo ./utils/searx.sh install packages
sudo ./utils/searxng.sh install packages
- name: Set up Python
uses: actions/setup-python@v2

6
.github/workflows/integration.yml

@ -19,7 +19,7 @@ jobs:
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: |
sudo ./utils/searx.sh install packages
sudo ./utils/searxng.sh install packages
sudo apt install firefox
- name: Set up Python
uses: actions/setup-python@v2
@ -55,7 +55,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
- name: Install Ubuntu packages
run: sudo ./utils/searx.sh install buildhost
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v2
with:
@ -82,7 +82,7 @@ jobs:
fetch-depth: '0'
persist-credentials: false
- name: Install Ubuntu packages
run: sudo ./utils/searx.sh install buildhost
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v2
with:

4
Makefile

@ -59,7 +59,6 @@ test.shell:
utils/brand.env \
$(MTOOLS) \
utils/lib.sh \
utils/lib_install.sh \
utils/lib_nvm.sh \
utils/lib_static.sh \
utils/lib_go.sh \
@ -69,8 +68,7 @@ test.shell:
utils/searxng.sh \
utils/morty.sh \
utils/lxc.sh \
utils/lxc-searxng.env \
.config.sh
utils/lxc-searxng.env
$(Q)$(MTOOLS) build_msg TEST "$@ OK"

2
manage

@ -417,8 +417,6 @@ docs.prebuild() {
[ "$VERBOSE" = "1" ] && set -x
mkdir -p "${DOCS_BUILD}/includes"
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
pyenv.cmd searxng_extra/docs_prebuild
)
dump_return $?

527
utils/filtron.sh

@ -4,56 +4,19 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/lib_go.sh
source "${REPO_ROOT}/utils/lib_go.sh"
# shellcheck source=utils/lib_install.sh
source "${REPO_ROOT}/utils/lib_install.sh"
# ----------------------------------------------------------------------------
# config
# ----------------------------------------------------------------------------
PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
FILTRON_ETC="/etc/filtron"
FILTRON_RULES="$FILTRON_ETC/rules.json"
FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/filtron/rules.json}"
FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
# The filtron target is the SearXNG installation, listenning on server.port at
# server.bind_address. The default of FILTRON_TARGET is taken from the YAML
# configuration, do not change this value without reinstalling the entire
# SearXNG suite including filtron & morty.
FILTRON_TARGET="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}"
SERVICE_NAME="filtron"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
GO_ENV="${SERVICE_HOME}/.go_env"
GO_VERSION="go1.17.2"
APACHE_FILTRON_SITE="searxng.conf"
NGINX_FILTRON_SITE="searxng.conf"
# shellcheck disable=SC2034
CONFIG_FILES=(
"${FILTRON_RULES}"
"${SERVICE_SYSTEMD_UNIT}"
)
APACHE_FILTRON_SITE="searx.conf"
NGINX_FILTRON_SITE="searx.conf"
# ----------------------------------------------------------------------------
usage() {
@ -62,248 +25,45 @@ usage() {
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|user|rules]
$(basename "$0") reinstall all
$(basename "$0") update [filtron]
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off]
$(basename "$0") apache [install|remove]
$(basename "$0") nginx [install|remove]
$(basename "$0") remove all]
$(basename "$0") apache remove
$(basename "$0") nginx remove
shell
start interactive shell from user ${SERVICE_USER}
install / remove
:all: complete setup of filtron service
:user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
:rules: reinstall filtron rules $FILTRON_RULES
install
:check: check the filtron installation
reinstall:
:all: runs 'install/remove all'
update filtron
Update filtron installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
show service status and log
option
set one of the available options
apache (${PUBLIC_URL})
:install: apache site with a reverse proxy (ProxyPass)
:remove: apache site ${APACHE_FILTRON_SITE}
nginx (${PUBLIC_URL})
:install: nginx site with a reverse proxy (ProxyPass)
:remove: nginx site ${NGINX_FILTRON_SITE}
filtron rules: ${FILTRON_RULES_TEMPLATE}
---- sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
FILTRON_TARGET : ${FILTRON_TARGET}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_URL_PATH : ${FILTRON_URL_PATH}
remove all : drop all components of the filtron service
apache remove : drop apache site ${APACHE_FILTRON_SITE}
nginx remove : drop nginx site ${NGINX_FILTRON_SITE}
EOF
install_log_searx_instance
[[ -n ${1} ]] && err_msg "$1"
}
main() {
required_commands \
sudo install git wget curl \
|| exit
local _usage="unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
reinstall)
rst_title "re-install $SERVICE_NAME" part
sudo_or_exit
case $2 in
all)
remove_all
install_all
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SERVICE_NAME" part
sudo_or_exit
case $2 in
check)
rst_title "Check filtron installation" part
install_check
;;
all) install_all ;;
user) assert_user ;;
rules)
install_rules
systemd_restart_service "${SERVICE_NAME}"
;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
filtron) update_filtron ;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service) systemd_activate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) systemd_deactivate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
install) install_nginx_site ;;
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
debug-on) echo; enable_debug ;;
debug-off) echo; disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc ;;
*) usage "unknown or missing command $1"; exit 42;;
esac
}
install_all() {
rst_title "Install $SERVICE_NAME (service)"
assert_user
wait_key
go.golang "${GO_VERSION}" "${SERVICE_USER}"
wait_key
install_filtron
install_rules
wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
wait_key
echo
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
elif nginx_is_installed; then
info_msg "nginx is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_nginx_site
fi
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
install_check() {
if service_account_is_available "$SERVICE_USER"; then
info_msg "service account $SERVICE_USER available."
else
err_msg "service account $SERVICE_USER not available!"
fi
if go_is_available "$SERVICE_USER"; then
info_msg "~$SERVICE_USER: go is installed"
else
err_msg "~$SERVICE_USER: go is not installed"
fi
if filtron_is_installed; then
info_msg "~$SERVICE_USER: filtron app is installed"
else
err_msg "~$SERVICE_USER: filtron app is not installed!"
fi
if ! service_is_available "http://${FILTRON_API}"; then
err_msg "API not available at: http://${FILTRON_API}"
fi
if ! service_is_available "http://${FILTRON_LISTEN}" ; then
err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}"
fi
if service_is_available "http://${FILTRON_TARGET}" ; then
info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
fi
if ! service_is_available "${PUBLIC_URL}"; then
warn_msg "Public service at ${PUBLIC_URL} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
if [[ "${GO_VERSION}" > "$(go_version)" ]]; then
warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least"
warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all"
else
info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)"
fi
if [ -f "${APACHE_SITES_AVAILABLE}/searx.conf" ]; then
warn_msg "old searx.conf apache site exists"
fi
if [ -f "${NGINX_APPS_AVAILABLE}/searx.conf" ]; then
warn_msg "old searx.conf nginx site exists"
fi
}
go_version(){
go.version "${SERVICE_USER}"
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
@ -321,219 +81,6 @@ installations that were installed with this script."
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
export SERVICE_HOME
echo "export SERVICE_HOME=$SERVICE_HOME"
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
touch "$GO_ENV"
grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile
EOF
}
filtron_is_installed() {
[[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
}
install_filtron() {
rst_title "Install filtron in user's ~/go-apps" section
echo
go.install github.com/searxng/filtron@latest "${SERVICE_USER}"
}
update_filtron() {
rst_title "Update filtron" section
echo
go.install github.com/searxng/filtron@latest "${SERVICE_USER}"
}
install_rules() {
rst_title "Install filtron rules"
echo
if [[ ! -f "${FILTRON_RULES}" ]]; then
info_msg "install rules ${FILTRON_RULES_TEMPLATE}"
info_msg " --> ${FILTRON_RULES}"
mkdir -p "$(dirname "${FILTRON_RULES}")"
cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
return
fi
if cmp --silent "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"; then
info_msg "${FILTRON_RULES} is up to date with"
info_msg "${FILTRON_RULES_TEMPLATE}"
return
fi
rst_para "Diff between origin's rules file (+) and current (-):"
echo "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"
$DIFF_CMD "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"
local action
choose_one action "What should happen to the rules file? " \
"keep configuration unchanged" \
"use origin rules" \
"start interactive shell"
case $action in
"keep configuration unchanged")
info_msg "leave rules file unchanged"
;;
"use origin rules")
backup_file "${FILTRON_RULES}"
info_msg "install origin rules"
cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
;;
"start interactive shell")
backup_file "${FILTRON_RULES}"
echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
sudo -H -i
rst_para 'Diff between new rules file (-) and current (+):'
echo
$DIFF_CMD "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
wait_key
;;
esac
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
FILTRON_TARGET : ${FILTRON_TARGET}
FILTRON_API : ${FILTRON_API}
FILTRON_LISTEN : ${FILTRON_LISTEN}
FILTRON_URL_PATH : ${FILTRON_URL_PATH}
EOF
install_log_searx_instance
install_check
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL}"
info_msg "internal URL --> http://${FILTRON_LISTEN}"
fi
local _debug_on
if ask_yn "Enable filtron debug mode?"; then
enable_debug
_debug_on=1
fi
echo
systemctl --no-pager -l status "${SERVICE_NAME}"
echo
info_msg "public URL --> ${PUBLIC_URL}"
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
journalctl -f -u "${SERVICE_NAME}"
done
if [[ $_debug_on == 1 ]]; then
disable_debug
fi
return 0
}
enable_debug() {
info_msg "try to enable debug mode ..."
python <<EOF
import sys, json
debug = {
u'name': u'debug request'
, u'filters': []
, u'interval': 0
, u'limit': 0
, u'actions': [{u'name': u'log'}]
}
with open('$FILTRON_RULES') as rules:
j = json.load(rules)
pos = None
for i in range(len(j)):
if j[i].get('name') == 'debug request':
pos = i
break
if pos is not None:
j[pos] = debug
else:
j.append(debug)
with open('$FILTRON_RULES', 'w') as rules:
json.dump(j, rules, indent=2, sort_keys=True)
EOF
systemctl restart "${SERVICE_NAME}.service"
}
disable_debug() {
info_msg "try to disable debug mode ..."
python <<EOF
import sys, json
with open('$FILTRON_RULES') as rules:
j = json.load(rules)
pos = None
for i in range(len(j)):
if j[i].get('name') == 'debug request':
pos = i
break
if pos is not None:
del j[pos]
with open('$FILTRON_RULES', 'w') as rules:
json.dump(j, rules, indent=2, sort_keys=True)
EOF
systemctl restart "${SERVICE_NAME}.service"
}
install_apache_site() {
rst_title "Install Apache site $APACHE_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
! apache_is_installed && info_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_apache_site() {
rst_title "Remove Apache site $APACHE_FILTRON_SITE"
@ -551,35 +98,6 @@ This removes apache site ${APACHE_FILTRON_SITE}."
}
install_nginx_site() {
rst_title "Install nginx site $NGINX_FILTRON_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
! nginx_is_installed && info_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_nginx
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
# shellcheck disable=SC2034
SEARXNG_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SRC)
# shellcheck disable=SC2034
SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH)
nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL}"; then
err_msg "Public service at ${PUBLIC_URL} is not available!"
fi
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_FILTRON_SITE"
@ -593,35 +111,8 @@ This removes nginx site ${NGINX_FILTRON_SITE}."
return
fi
nginx_remove_site "$FILTRON_FILTRON_SITE"
}
rst-doc() {
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
echo -e "\n.. START install systemd unit"
cat <<EOF
.. tabs::
.. group-tab:: systemd
.. code:: bash
EOF
eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
echo -e "\n.. END install systemd unit"
nginx_remove_app "$FILTRON_FILTRON_SITE"
# for DIST_NAME in ubuntu-20.04 arch fedora centos; do
# (
# DIST_ID=${DIST_NAME%-*}
# DIST_VERS=${DIST_NAME#*-}
# [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
# # ...
# )
# done
}
# ----------------------------------------------------------------------------

208
utils/lib_install.sh

@ -1,208 +0,0 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: AGPL-3.0-or-later
# https://github.com/koalaman/shellcheck/issues/356#issuecomment-853515285
# shellcheck source=utils/lib.sh
. /dev/null
# Initialize installation procedures:
#
# - Modified source_dot_config function that
# - loads .config.sh from an existing installation (at SEARXNG_SRC).
# - initialize **SEARX_SRC_INIT_FILES**
# - functions like:
# - install_log_searx_instance()
# - install_searx_get_state()
#
# usage:
# source lib_install.sh
#
# **Installation scripts**
#
# The utils/lib_install.sh is sourced by the installations scripts:
#
# - utils/searx.sh
# - utils/morty.sh
# - utils/filtron.sh
#
# If '${SEARXNG_SRC}/.config.sh' exists, the modified source_dot_config() function
# loads this configuration (instead of './.config.sh').
# **SEARX_SRC_INIT_FILES**
#
# Array of file names to sync into a installation at $SEARXNG_SRC. The file names
# are relative to the $REPO_ROOT. Set by function init_SEARXNG_SRC_INIT_FILES().
# Most often theses are files like:
# - .config.sh
# - searx/settings.yml
# - utils/brand.env
# - ...
SEARX_SRC_INIT_FILES=()
eval orig_"$(declare -f source_dot_config)"
source_dot_config() {
# Modified source_dot_config function that
# - loads .config.sh from an existing installation (at SEARXNG_SRC).
# - initialize SEARX_SRC_INIT_FILES
if [ -z "$eval_SEARXNG_SRC" ]; then
export eval_SEARXNG_SRC='true'
SEARXNG_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SRC)
SEARXNG_PYENV=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_PYENV)
SEARXNG_SETTINGS_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SETTINGS_PATH)
if [ ! -r "${SEARXNG_SRC}" ]; then
info_msg "not yet cloned: ${SEARXNG_SRC}"
orig_source_dot_config
return 0
fi
info_msg "using instance at: ${SEARXNG_SRC}"
# set and log DOT_CONFIG
if [ -r "${SEARXNG_SRC}/.config.sh" ]; then
info_msg "switching to ${SEARXNG_SRC}/.config.sh"
DOT_CONFIG="${SEARXNG_SRC}/.config.sh"
else
info_msg "using local config: ${DOT_CONFIG}"
fi
init_SEARX_SRC_INIT_FILES
fi
}
init_SEARX_SRC_INIT_FILES(){
# init environment SEARX_SRC_INIT_FILES
# Monitor modified files in the working-tree from the local repository, only
# if the local file differs to the corresponding file in the instance. Most
# often theses are files like:
#
# - .config.sh
# - searx/settings.yml
# - utils/brand.env
# - ...
# keep list empty if there is no installation
SEARX_SRC_INIT_FILES=()
if [ ! -r "$SEARXNG_SRC" ]; then
return 0
fi
local fname
local msg=""
local _prefix=""
if [[ -n ${SUDO_USER} ]]; then
_prefix="sudo -u ${SUDO_USER}"
fi
# Monitor local modified files from the repository, only if the local file
# differs to the corresponding file in the instance
while IFS= read -r fname; do
if [ -z "$fname" ]; then
continue
fi
if [ -r "${SEARXNG_SRC}/${fname}" ]; then
# diff "${REPO_ROOT}/${fname}" "${SEARXNG_SRC}/${fname}"
if ! cmp --silent "${REPO_ROOT}/${fname}" "${SEARXNG_SRC}/${fname}"; then
SEARX_SRC_INIT_FILES+=("${fname}")
info_msg "local clone (workingtree), modified file: ./$fname"
msg="to update use: sudo -H ./utils/searx.sh install init-src"
fi
fi
done <<< "$($_prefix git diff --name-only)"
[ -n "$msg" ] && info_msg "$msg"
}
install_log_searx_instance() {
echo -e "---- SearXNG instance setup ${_BBlue}(status: $(install_searx_get_state))${_creset}"
echo -e " SEARXNG_SETTINGS_PATH : ${_BBlue}${SEARXNG_SETTINGS_PATH}${_creset}"
echo -e " SEARXNG_PYENV : ${_BBlue}${SEARXNG_PYENV}${_creset}"
echo -e " SEARXNG_SRC : ${_BBlue}${SEARXNG_SRC:-none}${_creset}"
echo -e " SEARXNG_URL : ${_BBlue}${SEARXNG_URL:-none}${_creset}"
if in_container; then
# SearXNG is listening on 127.0.0.1 and not available from outside container
# in containers the service is listening on 0.0.0.0 (see lxc-searxng.env)
echo -e "---- container setup"
echo -e " ${_BBlack}HINT:${_creset} SearXNG only listen on loopback device" \
"${_BBlack}inside${_creset} the container."
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container (IPv6): [${ip#*|}]"
else
# IPv4:
echo " container (IPv4): ${ip#*|}"
fi
done
fi
}
install_searx_get_state(){
# usage: install_searx_get_state
#
# Prompts a string indicating the status of the installation procedure
#
# missing-searx-clone:
# There is no clone at ${SEARXNG_SRC}
# missing-searx-pyenv:
# There is no pyenv in ${SEARXNG_PYENV}
# installer-modified:
# There are files modified locally in the installer (clone),
# see ${SEARX_SRC_INIT_FILES} description.
# python-installed:
# Scripts can be executed in instance's environment
# - user: ${SERVICE_USER}
# - pyenv: ${SEARXNG_PYENV}
if [ -f /etc/searx/settings.yml ]; then
err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/"
fi
if ! [ -r "${SEARXNG_SRC}" ]; then
echo "missing-searx-clone"
return
fi
if ! [ -f "${SEARXNG_PYENV}/bin/activate" ]; then
echo "missing-searx-pyenv"
return
fi
if ! [ -r "${SEARXNG_SETTINGS_PATH}" ]; then
echo "missing-settings"
return
fi
if ! [ ${#SEARX_SRC_INIT_FILES[*]} -eq 0 ]; then
echo "installer-modified"
return
fi
echo "python-installed"
}
# Initialization of the installation procedure
# --------------------------------------------
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"
# SEARXNG_URL aka PUBLIC_URL: the public URL of the instance (e.g.
# "https://example.org/searx"). The value is taken from environment $SEARXNG_URL
# in ./utils/brand.env. This variable is a empty string if server.base_url in
# the settings.yml is set to 'false'.
SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)}"
if in_container; then
# hint: Linux containers do not have DNS entries, lets use IPs
SEARXNG_URL="http://$(primary_ip)"
fi
# shellcheck disable=SC2034
PUBLIC_URL="${SEARXNG_URL}"
source_dot_config
# shellcheck source=utils/lxc-searxng.env
source "${REPO_ROOT}/utils/lxc-searxng.env"
in_container && lxc_set_suite_env

1
utils/lxc.sh

@ -4,7 +4,6 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
source_dot_config
# shellcheck source=utils/brand.env
source "${REPO_ROOT}/utils/brand.env"

457
utils/morty.sh

@ -3,10 +3,6 @@
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
# shellcheck source=utils/lib_go.sh
source "${REPO_ROOT}/utils/lib_go.sh"
# shellcheck source=utils/lib_install.sh
source "${REPO_ROOT}/utils/lib_install.sh"
# ----------------------------------------------------------------------------
# config
@ -16,24 +12,9 @@ MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}"
PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}"
PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$PUBLIC_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}"
# shellcheck disable=SC2034
MORTY_TIMEOUT=5
SERVICE_NAME="morty"
SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
# shellcheck disable=SC2034
SERVICE_GROUP="${SERVICE_USER}"
# shellcheck disable=SC2034
SERVICE_ENV_DEBUG=false
GO_ENV="${SERVICE_HOME}/.go_env"
GO_VERSION="go1.17.2"
# shellcheck disable=SC2034
CONFIG_FILES=()
# Apache Settings
@ -47,267 +28,45 @@ usage() {
# shellcheck disable=SC1117
cat <<EOF
usage::
$(basename "$0") shell
$(basename "$0") install [all|check|user]
$(basename "$0") reinstall all
$(basename "$0") update [morty]
$(basename "$0") remove [all]
$(basename "$0") activate [service]
$(basename "$0") deactivate [service]
$(basename "$0") inspect [service]
$(basename "$0") option [debug-on|debug-off|new-key]
$(basename "$0") apache [install|remove]
$(basename "$0") nginx [install|remove]
$(basename "$0") info [searx]
$(basename "$0") remove all
$(basename "$0") apache remove
$(basename "$0") nginx remove
shell
start interactive shell from user ${SERVICE_USER}
install / remove
:all: complete setup of morty service
:user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
install
:check: check the morty installation
reinstall:
:all: runs 'install/remove all'
update morty
Update morty installation ($SERVICE_HOME)
activate service
activate and start service daemon (systemd unit)
deactivate service
stop and deactivate service daemon (systemd unit)
inspect service
show service status and log
option
set one of the available options
:new-key: set new morty key
apache : ${PUBLIC_URL_MORTY}
:install: apache site with a reverse proxy (ProxyPass)
:remove: apache site ${APACHE_MORTY_SITE}
nginx (${PUBLIC_URL_MORTY})
:install: nginx site with a reverse proxy (ProxyPass)
:remove: nginx site ${NGINX_MORTY_SITE}
----
sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY}
MORTY_LISTEN: : ${MORTY_LISTEN}
remove all : drop all components of the morty service
apache remove : drop apache site ${APACHE_MORTY_SITE}
nginx remove : drop nginx site ${NGINX_MORTY_SITE}
EOF
install_log_searx_instance
if in_container; then
# in containers the service is listening on 0.0.0.0 (see lxc-searxng.env)
for ip in $(global_IPs) ; do
if [[ $ip =~ .*:.* ]]; then
echo " container URL (IPv6): http://[${ip#*|}]:3000/"
else
# IPv4:
echo " container URL (IPv4): http://${ip#*|}:3000/"
fi
done
fi
echo
info_searx
[[ -n ${1} ]] && err_msg "$1"
}
info_searx() {
# shellcheck disable=SC1117
cat <<EOF
To activate result and image proxy in SearXNG read:
https://docs.searxng.org/admin/morty.html
Check settings in file ${SEARXNG_SETTINGS_PATH} ...
result_proxy:
url : ${PUBLIC_URL_MORTY}
server:
image_proxy : True
EOF
}
main() {
required_commands \
sudo install git wget curl \
|| exit
local _usage="ERROR: unknown or missing $1 command $2"
case $1 in
--getenv) var="$2"; echo "${!var}"; exit 0;;
-h|--help) usage; exit 0;;
shell)
sudo_or_exit
interactive_shell "${SERVICE_USER}"
;;
inspect)
case $2 in
service)
sudo_or_exit
inspect_service
;;
*) usage "$_usage"; exit 42;;
esac ;;
reinstall)
rst_title "re-install $SERVICE_NAME" part
sudo_or_exit
case $2 in
all)
remove_all
install_all
;;
*) usage "$_usage"; exit 42;;
esac ;;
install)
rst_title "$SERVICE_NAME" part
sudo_or_exit
case $2 in
all) install_all ;;
check)
rst_title "Check morty installation" part
install_check
;;
user) assert_user ;;
*) usage "$_usage"; exit 42;;
esac ;;
update)
sudo_or_exit
case $2 in
morty) update_morty ;;
*) usage "$_usage"; exit 42;;
esac ;;
remove)
sudo_or_exit
case $2 in
all) remove_all;;
user) drop_service_account "${SERVICE_USER}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
activate)
sudo_or_exit
case $2 in
service) systemd_activate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
deactivate)
sudo_or_exit
case $2 in
service) systemd_deactivate_service "${SERVICE_NAME}" ;;
*) usage "$_usage"; exit 42;;
esac ;;
apache)
sudo_or_exit
case $2 in
install) install_apache_site ;;
remove) remove_apache_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
nginx)
sudo_or_exit
case $2 in
install) install_nginx_site ;;
remove) remove_nginx_site ;;
*) usage "$_usage"; exit 42;;
esac ;;
info)
case $2 in
searx) info_searx ;;
*) usage "$_usage"; exit 42;;
esac ;;
option)
sudo_or_exit
case $2 in
new-key) set_new_key ;;
debug-on) enable_debug ;;
debug-off) disable_debug ;;
*) usage "$_usage"; exit 42;;
esac ;;
doc) rst-doc ;;
*) usage "ERROR: unknown or missing command $1"; exit 42;;
esac
}
install_all() {
MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
rst_title "Install $SERVICE_NAME (service)"
assert_user
wait_key
go.golang "${GO_VERSION}" "${SERVICE_USER}"
wait_key
install_morty
wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
wait_key
if ! service_is_available "http://${MORTY_LISTEN}" ; then
err_msg "Morty is not listening on: http://${MORTY_LISTEN}"
fi
if apache_is_installed; then
info_msg "Apache is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_apache_site
fi
elif nginx_is_installed; then
info_msg "nginx is installed on this host."
if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
install_nginx_site
fi
fi
info_searx
if ask_yn "Add image and result proxy to SearXNG settings.yml?" Yn; then
"${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
"${REPO_ROOT}/utils/searx.sh" option image-proxy-on
fi
if ask_yn "Do you want to inspect the installation?" Ny; then
inspect_service
fi
}
install_check() {
if service_account_is_available "$SERVICE_USER"; then
info_msg "service account $SERVICE_USER available."
else
err_msg "service account $SERVICE_USER not available!"
fi
if go_is_available "$SERVICE_USER"; then
info_msg "~$SERVICE_USER: go is installed"
else
err_msg "~$SERVICE_USER: go is not installed"
fi
if morty_is_installed; then
info_msg "~$SERVICE_USER: morty app is installed"
else
err_msg "~$SERVICE_USER: morty app is not installed!"
fi
if ! service_is_available "http://${MORTY_LISTEN}" ; then
err_msg "Morty is not listening on: http://${MORTY_LISTEN}"
echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .."
wait_key
fi
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
if ! in_container; then
warn_msg "Check if public name is correct and routed or use the public IP from above."
fi
fi
if [[ "${GO_VERSION}" > "$(go_version)" ]]; then
warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least"
warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all"
else
info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)"
fi
}
go_version(){
go.version "${SERVICE_USER}"
}
remove_all() {
rst_title "De-Install $SERVICE_NAME (service)"
@ -321,152 +80,6 @@ installations that were installed with this script."
fi
}
assert_user() {
rst_title "user $SERVICE_USER" section
echo
tee_stderr 1 <<EOF | bash | prefix_stdout
useradd --shell /bin/bash --system \
--home-dir "$SERVICE_HOME" \
--comment 'Web content sanitizer proxy' $SERVICE_USER
mkdir "$SERVICE_HOME"
chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
groups $SERVICE_USER
EOF
SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
export SERVICE_HOME
echo "export SERVICE_HOME=$SERVICE_HOME"
tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
touch $GO_ENV
grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile
EOF
}
morty_is_installed() {
[[ -f $SERVICE_HOME/go-apps/bin/morty ]]
}
install_morty() {
rst_title "Install morty in user's ~/go-apps" section
echo
go.install github.com/asciimoo/morty@latest "${SERVICE_USER}"
}
update_morty() {
rst_title "Update morty" section
echo
go.install github.com/asciimoo/morty@latest "${SERVICE_USER}"
}
set_service_env_debug() {
# usage: set_service_env_debug [false|true]
# shellcheck disable=SC2034
local SERVICE_ENV_DEBUG="${1:-false}"
if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
fi
}
inspect_service() {
rst_title "service status & log"
cat <<EOF
sourced ${DOT_CONFIG} :
SERVICE_USER : ${SERVICE_USER}
SERVICE_HOME : ${SERVICE_HOME}
PUBLIC_URL_MORTY: : ${PUBLIC_URL_MORTY}
MORTY_LISTEN: : ${MORTY_LISTEN}
EOF
install_log_searx_instance
install_check
if in_container; then
lxc_suite_info
else
info_msg "public URL --> ${PUBLIC_URL_MORTY}"
info_msg "morty URL --> http://${MORTY_LISTEN}"
fi
local _debug_on
if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then
enable_debug
_debug_on=1
else
systemctl --no-pager -l status "${SERVICE_NAME}"
fi
echo
# shellcheck disable=SC2059
printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
read -r -s -n1 -t 5
echo
while true; do
trap break 2
journalctl -f -u "${SERVICE_NAME}"
done
if [[ $_debug_on == 1 ]]; then
FORCE_SELECTION=Y disable_debug
fi
return 0
}
enable_debug() {
warn_msg "Do not enable debug in production environments!!"
info_msg "Enabling debug option needs to reinstall systemd service!"
set_service_env_debug true
}
disable_debug() {
info_msg "Disabling debug option needs to reinstall systemd service!"
set_service_env_debug false
}
set_new_key() {
rst_title "Set morty key"
echo
MORTY_KEY="$(head -c 32 /dev/urandom | base64)"
info_msg "morty key: '${MORTY_KEY}'"
warn_msg "this will need to reinstall services .."
MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key
systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
"${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}"
"${REPO_ROOT}/utils/searx.sh" option image-proxy-on
}
install_apache_site() {
rst_title "Install Apache site $APACHE_MORTY_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})"
! apache_is_installed && err_msg "Apache is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_apache
fi
apache_install_site "${APACHE_MORTY_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
fi
}
remove_apache_site() {
@ -484,35 +97,6 @@ This removes apache site ${APACHE_MORTY_SITE}."
apache_remove_site "$APACHE_MORTY_SITE"
}
install_nginx_site() {
rst_title "Install nginx site $NGINX_MORTY_SITE"
rst_para "\
This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})"
! nginx_is_installed && err_msg "nginx is not installed."
if ! ask_yn "Do you really want to continue?" Yn; then
return
else
install_nginx
fi
"${REPO_ROOT}/utils/searx.sh" install uwsgi
# shellcheck disable=SC2034
SEARXNG_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_SRC)
# shellcheck disable=SC2034
SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH)
nginx_install_app "${NGINX_MORTY_SITE}"
info_msg "testing public url .."
if ! service_is_available "${PUBLIC_URL_MORTY}"; then
err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!"
fi
}
remove_nginx_site() {
rst_title "Remove nginx site $NGINX_MORTY_SITE"
@ -526,37 +110,10 @@ This removes nginx site ${NGINX_MORTY_SITE}."
return
fi
nginx_remove_site "$NGINX_MORTY_SITE"
}
rst-doc() {
eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\""
echo -e "\n.. START install systemd unit"
cat <<EOF
.. tabs::
.. group-tab:: systemd
nginx_remove_app "$NGINX_MORTY_SITE"
.. code:: bash
EOF
eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "