Add files via upload

rc_scripts/iredadmin.debian

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Author: Zhang Huangbin (zhb@iredmail.org)
+
+### BEGIN INIT INFO
+# Provides:          api-server
+# Required-Start:    $network $syslog
+# Required-Stop:     $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: iredadmin instance
+# Description: iredadmin
+### END INIT INFO
+
+PROG='iredadmin'
+PIDFILE='/var/run/iredadmin/iredadmin.pid'
+UWSGI_INI_FILE='/opt/www/iredadmin/rc_scripts/uwsgi/debian.ini'
+
+check_status() {
+    # Usage: check_status pid_number
+    PID="${1}"
+    l=$(ps -p ${PID} | wc -l | awk '{print $1}')
+    if [ X"$l" == X"2" ]; then
+        echo "running"
+    else
+        echo "stopped"
+    fi
+}
+
+start() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "${PROG} is already running."
+            exit 0
+        else
+            rm -f ${PIDFILE} >/dev/null 2>&1
+        fi
+
+        unset s
+    fi
+
+    mkdir /var/run/iredadmin 2>/dev/null
+    chown iredadmin:iredadmin /var/run/iredadmin
+    chmod 0755 /var/run/iredadmin
+
+    echo "Starting ${PROG} ..."
+    uwsgi -d \
+        --ini ${UWSGI_INI_FILE} \
+        --pidfile ${PIDFILE} \
+        --log-syslog
+}
+
+stop() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "Stopping ${PROG} ..."
+            uwsgi --stop ${PIDFILE}
+            if [ X"$?" == X"0" ]; then
+                rm -f ${PIDFILE} >/dev/null 2>&1
+                rm -rf /var/run/iredadmin
+            else
+                echo -e "\t\t[ FAILED ]"
+            fi
+        else
+            echo "${PROG} is already stopped."
+            rm -f ${PIDFILE} >/dev/null 2>&1
+        fi
+    else
+        echo "${PROG} is already stopped."
+    fi
+    unset s
+}
+
+status() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "${PROG} is running."
+            exit 0
+        else
+            echo "${PROG} is stopped."
+            exit 1
+        fi
+    else
+        echo "${PROG} is stopped."
+        exit 3
+    fi
+}
+
+case "$1" in
+    start) start ;;
+    stop) stop ;;
+    status) status ;;
+    restart) stop && start ;;
+    *)
+        echo $"Usage: $0 {start|stop|restart|status}"
+        RETVAL=1
+        ;;
+esac

rc_scripts/iredadmin.freebsd

@@ -0,0 +1,110 @@
+#!/bin/sh
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+
+# PROVIDE: iredadmin
+# REQUIRE: DAEMON
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+name='iredadmin'
+rcvar=`set_rcvar_obsolete`
+start_precmd="iredadmin_precmd"
+
+RUN_DIR='/var/run/iredadmin'
+PIDFILE="${RUN_DIR}/iredadmin.pid"
+UWSGI_INI_FILE='/opt/www/iredadmin/rc_scripts/uwsgi/freebsd.ini'
+
+PATH="/usr/local/bin:/usr/local/sbin:$PATH"
+
+iredadmin_precmd() {
+    /usr/bin/install -m 0644 -o iredadmin -g iredadmin -d ${RUN_DIR}
+}
+
+check_status() {
+    # Usage: check_status pid_number
+    PID="${1}"
+    l=$(ps -p ${PID} | wc -l | awk '{print $1}')
+    if [ X"$l" == X"2" ]; then
+        echo "running"
+    else
+        echo "stopped"
+    fi
+}
+
+start() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "${name} is already running."
+        else
+            rm -f ${PIDFILE} >/dev/null 2>&1
+        fi
+
+        unset s
+    fi
+
+    /bin/mkdir $(dirname ${PIDFILE}) 2>/dev/null
+    /usr/sbin/chown iredadmin:iredadmin $(dirname ${PIDFILE})
+
+    echo "Starting ${name}."
+    uwsgi --ini ${UWSGI_INI_FILE} \
+          --pidfile ${PIDFILE} \
+          --log-syslog \
+          --daemonize /dev/null
+}
+
+stop() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "Stopping ${name}."
+            uwsgi --stop ${PIDFILE}
+            if [ X"$?" == X"0" ]; then
+                rm -f ${PIDFILE} >/dev/null 2>&1
+            else
+                echo -e "\t\t[ FAILED ]"
+            fi
+        else
+            echo "${name} is already stopped."
+            rm -f ${PIDFILE} >/dev/null 2>&1
+        fi
+
+        unset s
+    else
+        echo "${name} is already stopped."
+    fi
+}
+
+status() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "${name} is running."
+            exit 0
+        else
+            echo "${name} is stopped."
+            exit 1
+        fi
+
+        unset s
+    else
+        echo "${name} is stopped."
+        exit 3
+    fi
+}
+
+start_cmd="start"
+stop_cmd="stop"
+status_cmd="status"
+restart_cmd="stop && sleep 2 && start"
+
+command="start"
+load_rc_config ${name}
+run_rc_command "$1"

rc_scripts/iredadmin.openbsd

@@ -0,0 +1,23 @@
+#!/bin/ksh
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Start/stop iRedAdmin uwsgi instance.
+
+RUN_DIR='/var/run/iredadmin'
+PID_FILE="${RUN_DIR}/iredadmin.pid"
+UWSGI_INI_FILE='/opt/www/iredadmin/rc_scripts/uwsgi/openbsd.ini'
+
+daemon="/usr/local/bin/uwsgi --ini ${UWSGI_INI_FILE} --log-syslog --pidfile ${PID_FILE} --daemonize /dev/null"
+daemon_user='iredadmin'
+daemon_group='iredadmin'
+
+. /etc/rc.d/rc.subr
+
+rc_pre() {
+    install -d -o ${daemon_user} -g ${daemon_group} -m 0775 ${RUN_DIR}
+}
+
+rc_stop() {
+    kill -INT `cat ${PID_FILE}`
+}
+
+rc_cmd $1

rc_scripts/iredadmin.rhel

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+# Author: Zhang Huangbin (zhb@iredmail.org)
+
+### BEGIN INIT INFO
+# chkconfig: - 99 99
+# description: iredadmin instance
+# processname: iredadmin
+### END INIT INFO
+
+PROG='iredadmin'
+BINPATH='/opt/www/iredadmin/iredadmin.py'
+PIDFILE='/var/run/iredadmin/iredadmin.pid'
+UWSGI_INI_FILE='/opt/www/iredadmin/rc_scripts/uwsgi/rhel.ini'
+
+check_status() {
+    # Usage: check_status pid_number
+    PID="${1}"
+    l=$(ps -p ${PID} | wc -l | awk '{print $1}')
+    if [ X"$l" == X"2" ]; then
+        echo "running"
+    else
+        echo "stopped"
+    fi
+}
+
+start() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "${PROG} is already running."
+        else
+            rm -f ${PIDFILE} >/dev/null 2>&1
+        fi
+    fi
+
+    unset s
+
+    mkdir /var/run/iredadmin 2>/dev/null
+    chown iredadmin:iredadmin /var/run/iredadmin
+    chmod 0755 /var/run/iredadmin
+
+    echo "Starting ${PROG} ..."
+    uwsgi -d \
+        --ini ${UWSGI_INI_FILE} \
+        --pidfile ${PIDFILE} \
+        --log-syslog
+}
+
+stop() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "Stopping ${PROG} ..."
+            kill -9 ${PID}
+            if [ X"$?" == X"0" ]; then
+                rm -f ${PIDFILE} >/dev/null 2>&1
+                rm -rf /var/run/iredadmin
+            else
+                echo -e "\t\t[ FAILED ]"
+            fi
+        else
+            echo "${PROG} is already stopped."
+            rm -f ${PIDFILE} >/dev/null 2>&1
+        fi
+    else
+        echo "${PROG} is already stopped."
+    fi
+
+    unset s
+}
+
+status() {
+    if [ -f ${PIDFILE} ]; then
+        PID="$(cat ${PIDFILE})"
+        s="$(check_status ${PID})"
+
+        if [ X"$s" == X"running" ]; then
+            echo "${PROG} is running."
+            exit 0
+        else
+            echo "${PROG} is stopped."
+            exit 1
+        fi
+    else
+        echo "${PROG} is stopped."
+        exit 3
+    fi
+}
+
+case "$1" in
+    start) start ;;
+    stop) stop ;;
+    status) status ;;
+    restart) stop && sleep 1 && start ;;
+    *)
+        echo $"Usage: $0 {start|stop|restart|status}"
+        RETVAL=1
+        ;;
+esac

rc_scripts/systemd/debian.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=iRedAdmin daemon service
+After=network.target local-fs.target remote-fs.target
+
+[Service]
+Type=simple
+ExecStartPre=-/bin/mkdir -p /var/run/iredadmin
+ExecStartPre=/bin/chown iredadmin:iredadmin /var/run/iredadmin
+ExecStartPre=/bin/chmod 0755 /var/run/iredadmin
+ExecStart=/usr/bin/uwsgi --ini /opt/www/iredadmin/rc_scripts/uwsgi/debian.ini --pidfile /var/run/iredadmin/iredadmin.pid
+ExecStop=/usr/bin/uwsgi --stop /var/run/iredadmin/iredadmin.pid
+ExecStopPost=/bin/rm -rf /var/run/iredadmin
+KillSignal=SIGTERM
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

rc_scripts/systemd/rhel7.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=iRedAdmin daemon service
+After=network.target local-fs.target remote-fs.target
+
+[Service]
+Type=simple
+ExecStartPre=-/usr/bin/mkdir /var/run/iredadmin
+ExecStartPre=/usr/bin/chown iredadmin:iredadmin /var/run/iredadmin
+ExecStartPre=/usr/bin/chmod 0755 /var/run/iredadmin
+ExecStart=/usr/sbin/uwsgi --ini /opt/www/iredadmin/rc_scripts/uwsgi/rhel7.ini --pidfile /var/run/iredadmin/iredadmin.pid
+ExecStop=/usr/sbin/uwsgi --stop /var/run/iredadmin/iredadmin.pid
+ExecStopPost=/usr/bin/rm -rf /var/run/iredadmin
+KillSignal=SIGTERM
+TimeoutStopSec=5
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

rc_scripts/systemd/rhel8.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=iRedAdmin daemon service
+After=network.target local-fs.target remote-fs.target
+
+[Service]
+Type=simple
+ExecStartPre=-/usr/bin/mkdir /var/run/iredadmin
+ExecStartPre=/usr/bin/chown iredadmin:iredadmin /var/run/iredadmin
+ExecStartPre=/usr/bin/chmod 0755 /var/run/iredadmin
+ExecStart=/usr/local/bin/uwsgi --ini /opt/www/iredadmin/rc_scripts/uwsgi/rhel8.ini --pidfile /var/run/iredadmin/iredadmin.pid
+ExecStop=/usr/local/bin/uwsgi --stop /var/run/iredadmin/iredadmin.pid
+ExecStopPost=/usr/bin/rm -rf /var/run/iredadmin
+KillSignal=SIGTERM
+TimeoutStopSec=5
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

rc_scripts/systemd/rhel9.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=iRedAdmin daemon service
+After=network.target local-fs.target remote-fs.target
+
+[Service]
+Type=simple
+ExecStartPre=-/usr/bin/mkdir /var/run/iredadmin
+ExecStartPre=/usr/bin/chown iredadmin:iredadmin /var/run/iredadmin
+ExecStartPre=/usr/bin/chmod 0755 /var/run/iredadmin
+ExecStart=/usr/sbin/uwsgi --ini /opt/www/iredadmin/rc_scripts/uwsgi/rhel9.ini --pidfile /var/run/iredadmin/iredadmin.pid
+ExecStop=/usr/sbin/uwsgi --stop /var/run/iredadmin/iredadmin.pid
+ExecStopPost=/usr/bin/rm -rf /var/run/iredadmin
+KillSignal=SIGTERM
+TimeoutStopSec=5
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

rc_scripts/uwsgi/debian.ini

@@ -0,0 +1,17 @@
+[uwsgi]
+plugins = python3,syslog
+master = true
+vhost = true
+enable-threads = true
+processes = 5
+buffer-size = 8192
+logger = syslog:iredadmin,local5
+log-format = [%(addr)] %(method) %(uri) %(status) %(size) "%(referer)"
+
+uwsgi-socket = 127.0.0.1:7791
+
+uid = iredadmin
+gid = iredadmin
+
+chdir = /opt/www/iredadmin
+wsgi-file = iredadmin.py

rc_scripts/uwsgi/freebsd.ini

@@ -0,0 +1,20 @@
+[uwsgi]
+master = true
+vhost = true
+enable-threads = true
+processes = 5
+buffer-size = 8192
+logger = syslog:iredadmin,local5
+log-format = [%(addr)] %(method) %(uri) %(status) %(size) "%(referer)"
+
+# Log pid of master process
+safe-pid = true
+pidfile = /var/run/iredadmin/iredadmin.pid
+
+uwsgi-socket = 127.0.0.1:7791
+
+uid = iredadmin
+gid = iredadmin
+
+chdir = /usr/local/www/iredadmin
+wsgi-file = iredadmin.py

rc_scripts/uwsgi/openbsd.ini

@@ -0,0 +1,13 @@
+[uwsgi]
+master = true
+vhost = true
+enable-threads = true
+processes = 5
+buffer-size = 8192
+logger = syslog:iredadmin,local5
+log-format = [%(addr)] %(method) %(uri) %(status) %(size) "%(referer)"
+
+uwsgi-socket = 127.0.0.1:7791
+
+chdir = /var/www/iredadmin
+wsgi-file = iredadmin.py

rc_scripts/uwsgi/rhel7.ini

@@ -0,0 +1,17 @@
+[uwsgi]
+plugins = python36,syslog
+master = true
+vhost = true
+enable-threads = true
+processes = 5
+buffer-size = 8192
+logger = syslog:iredadmin,local5
+log-format = [%(addr)] %(method) %(uri) %(status) %(size) "%(referer)"
+
+uwsgi-socket = 127.0.0.1:7791
+
+uid = iredadmin
+gid = iredadmin
+
+chdir = /opt/www/iredadmin
+wsgi-file = iredadmin.py

rc_scripts/uwsgi/rhel8.ini

@@ -0,0 +1,17 @@
+[uwsgi]
+plugins = python3,syslog
+master = true
+vhost = true
+enable-threads = true
+processes = 5
+buffer-size = 8192
+logger = syslog:iredadmin,local5
+log-format = [%(addr)] %(method) %(uri) %(status) %(size) "%(referer)"
+
+uwsgi-socket = 127.0.0.1:7791
+
+uid = iredadmin
+gid = iredadmin
+
+chdir = /opt/www/iredadmin
+wsgi-file = iredadmin.py

rc_scripts/uwsgi/rhel9.ini

@@ -0,0 +1,17 @@
+[uwsgi]
+plugins = python3,syslog
+master = true
+vhost = true
+enable-threads = true
+processes = 5
+buffer-size = 8192
+logger = syslog:iredadmin,local5
+log-format = [%(addr)] %(method) %(uri) %(status) %(size) "%(referer)"
+
+uwsgi-socket = 127.0.0.1:7791
+
+uid = iredadmin
+gid = iredadmin
+
+chdir = /opt/www/iredadmin
+wsgi-file = iredadmin.py

tools/README.md

@@ -0,0 +1,44 @@
+# Cron Jobs
+
+* dump_disclaimer.py
+
+    Dump per-domain disclaimer which stored in LDAP or SQL database.
+    It's safe to execute it manually.
+
+* cleanup_amavisd_db.py
+
+    Cleanup old records from Amavisd database. It's safe to execute it manually.
+
+* delete_mailboxes.py
+
+    Delete mailboxes which are scheduled to be removed. The schedule date
+    was set while you removed the mail account with iRedAdmin(-Pro).
+
+# Utils
+
+* upgrade_iredadmin.sh
+
+    Upgrade an old iRedAdmin-Pro or iRedAdmin open source edition to current
+    release.
+
+* update_mailbox_quota.py
+
+    Update mailbox quota for one user (specified on command line) or bulk users
+    (read from a plain text file).
+
+* notify_quarantined_recipients.py
+
+    Notify local recipients (via email) that they have emails quarantined on
+    server and not delivered to their mailbox.
+
+* convert_ini_to_py.sh
+
+    Convert old iRedAdmin-Pro config file (.ini format) to the new one.
+
+* migrate_cluebringer_wblist_to_amavisd.py
+
+    Migrate Cluebringer white/blacklists to Amavisd database, and, optionally,
+    delete them in Cluebringer database.
+
+    Note: Don't forget to enable iRedAPD plugin `amavisd_wblist` in
+    `/opt/iredapd/settings.py`.

tools/cleanup_amavisd_db.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Remove old records in Amavisd database.
+
+# USAGE:
+#
+#   1: Make sure you have correct database settings in iRedAdmin config file
+#      'settings.py' for Amavisd.
+#
+#   2: Make sure you have proper values for below two parameters:
+#
+#       AMAVISD_REMOVE_MAILLOG_IN_DAYS = 7
+#       AMAVISD_REMOVE_QUARANTINED_IN_DAYS = 7
+#
+#      Default values is defined in libs/default_settings.py, you can override
+#      them in settings.py. WARNING: DO NOT MODIFY libs/default_settings.py.
+#
+#   3: Test this script in command line directly, make sure no errors in output
+#      message.
+#
+#       # python cleanup_amavisd_db.py
+#
+#   4: Setup a daily cron job to execute this script. For example: execute
+#      it daily at 1:30AM.
+#
+#       30  1   *   *   *   python /path/to/cleanup_amavisd_db.py >/dev/null
+#
+# That's all.
+
+import os
+import sys
+import time
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from libs import iredutils
+from tools import ira_tool_lib
+
+web.config.debug = ira_tool_lib.debug
+logger = ira_tool_lib.logger
+
+if not (settings.amavisd_enable_logging or settings.amavisd_enable_quarantine):
+    sys.exit("Amavisd is not enabled. SKIP.")
+
+backend = settings.backend
+logger.info('Backend: %s' % backend)
+logger.info('SQL server: %s:%d' % (settings.amavisd_db_host, int(settings.amavisd_db_port)))
+
+db_settings = iredutils.get_settings_from_db(params=['amavisd_remove_quarantined_in_days', 'amavisd_remove_maillog_in_days'])
+keep_quar_days = db_settings['amavisd_remove_quarantined_in_days']
+keep_inout_days = db_settings['amavisd_remove_maillog_in_days']
+query_size_limit = settings.AMAVISD_CLEANUP_QUERY_SIZE_LIMIT
+
+# SQL records in `quarantine` table reference to `msgs`.
+if keep_quar_days > keep_inout_days:
+    keep_inout_days = keep_quar_days
+
+conn_amavisd = ira_tool_lib.get_db_conn('amavisd')
+
+if settings.backend in ['mysql', 'ldap']:
+    # Querying (SELECT) without locking. Require MySQL 5.0+ and InnoDB.
+    #
+    # Since we're dealing with sql records created days ago, no new records
+    # will be inserted with that date, it's safe to use dirty read.
+    logger.info('Enable dirty read for querying without locking SQL tables.')
+    try:
+        conn_amavisd.query('SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED')
+    except Exception as e:
+        logger.error('Cannot enable dirty read: %s' % repr(e))
+
+
+# Removing records from single table.
+def remove_from_one_table(sql_table, index_column, removed_values):
+    total = len(removed_values)
+
+    # Delete how many records each time
+    offset = query_size_limit
+
+    if total:
+        loop_times = total / offset
+        if total % offset:
+            loop_times += 1
+
+        for i in range(int(loop_times)):
+            removing_values = removed_values[offset * i: offset * (i + 1)]
+            logger.info(
+                '\t[-] Deleting records: %d - %d (%s)' % (i * offset, i * offset + len(removing_values), time.ctime()))
+            conn_amavisd.delete(sql_table,
+                                vars={'ids': removing_values},
+                                where='%s IN $ids' % index_column)
+
+
+# Delete old quarantined mails from table 'msgs'. It will also
+# delete records in table 'quarantine'.
+logger.info('Delete quarantined mails which older than %d days' % keep_quar_days)
+_now = int(time.time())
+_expire_seconds = _now - (keep_quar_days * 86400)
+sql_where = """time_num < %d AND quar_type='Q'""" % _expire_seconds
+
+counter_msgs = 0
+while True:
+    qr = conn_amavisd.select('msgs',
+                             what='mail_id',
+                             where=sql_where,
+                             limit=query_size_limit)
+
+    if qr:
+        ids = [r.mail_id for r in qr]
+        _total = len(ids)
+
+        logger.info('\t[-] Deleting records: %d - %d (%s)' % (counter_msgs + 1, counter_msgs + _total, time.ctime()))
+
+        conn_amavisd.delete('msgs', vars={'ids': ids}, where='mail_id IN $ids')
+        conn_amavisd.delete('msgrcpt', vars={'ids': ids}, where='mail_id IN $ids')
+
+        counter_msgs += len(ids)
+    else:
+        break
+
+logger.info('Delete incoming/outgoing emails which older than %d days' % keep_inout_days)
+
+_now = int(time.time())
+_expire_seconds = _now - (keep_inout_days * 86400)
+sql_where = """time_num < %d AND (quar_type <> 'Q' OR quar_type IS NULL)""" % _expire_seconds
+
+# We experienced an issue with PostgreSQL, it always return an non-existing
+# SQL record, and it causes endless loop. As a hack, we store all removed
+# `mail_id` and compare new `mail_id` with this list.
+_removed_ids = set()
+
+counter_msgrcpt = 0
+while True:
+    qr = conn_amavisd.select('msgs',
+                             what='mail_id',
+                             where=sql_where,
+                             limit=query_size_limit)
+
+    if qr:
+        ids = [iredutils.bytes2str(r.mail_id) for r in qr]
+        _total = len(ids)
+
+        _removing_ids = list(set(ids) - set(_removed_ids))
+        if not _removing_ids:
+            break
+
+        logger.info(
+            '\t[-] Deleting records: %d - %d (%s)' % (counter_msgrcpt + 1, counter_msgrcpt + _total, time.ctime()))
+
+        conn_amavisd.delete('msgs', vars={'ids': _removing_ids}, where='mail_id IN $ids')
+        conn_amavisd.delete('msgrcpt', vars={'ids': _removing_ids}, where='mail_id IN $ids')
+
+        counter_msgrcpt += _total
+        _removed_ids.update(ids)
+    else:
+        break
+
+# delete unreferenced records from tables msgrcpt, quarantine and maddr
+logger.info('Delete unreferenced records from table `msgrcpt`.')
+conn_amavisd.query('''
+    DELETE FROM msgrcpt
+    WHERE NOT EXISTS (SELECT 1 FROM msgs WHERE mail_id=msgrcpt.mail_id)
+''')
+
+#
+# Delete unreferenced records from table `quarantine`.
+#
+logger.info('Delete unreferenced records from table `quarantine`.')
+msgs_mail_ids = set()
+maddr_ids_in_use = set()
+quar_mail_ids = set()
+
+qr = conn_amavisd.select('msgs', what='mail_id, sid')
+for i in qr:
+    msgs_mail_ids.add(i.mail_id)
+    maddr_ids_in_use.add(i.sid)
+
+qr = conn_amavisd.select('quarantine', what='mail_id')
+for i in qr:
+    quar_mail_ids.add(i.mail_id)
+
+invalid_quar_mail_ids = [i for i in quar_mail_ids if i not in msgs_mail_ids]
+remove_from_one_table(sql_table='quarantine',
+                      index_column='mail_id',
+                      removed_values=invalid_quar_mail_ids)
+
+#
+# Delete unreferenced records from table `maddr`.
+#
+logger.info('Delete unreferenced records from table `maddr`.')
+
+# Get all maddr.id
+maddr_ids = set()
+qr = conn_amavisd.select('maddr', what='id')
+for i in qr:
+    maddr_ids.add(i.id)
+
+qr = conn_amavisd.select('msgrcpt', what='rid')
+for i in qr:
+    maddr_ids_in_use.add(i.rid)
+
+invalid_maddr_ids = [i for i in maddr_ids if i not in maddr_ids_in_use]
+remove_from_one_table(sql_table='maddr',
+                      index_column='id',
+                      removed_values=invalid_maddr_ids)
+
+#
+# Delete unreferenced records from table `mailaddr`.
+#
+logger.info('Delete unreferenced records from table `mailaddr`.')
+
+# Get all `mailaddr.id`
+mailaddr_ids = set()
+qr = conn_amavisd.select('mailaddr', what='id')
+for i in qr:
+    mailaddr_ids.add(i.id)
+
+# Get all `wblist.sid` and `outbound_wblist.rid` (both refer to `mailaddr.id`)
+wblist_ids = set()
+
+qr = conn_amavisd.select('wblist', what='sid')
+for i in qr:
+    wblist_ids.add(i.sid)
+
+try:
+    qr = conn_amavisd.select('outbound_wblist', what='rid')
+    for i in qr:
+        wblist_ids.add(i.rid)
+except:
+    # No outbound_wblist table
+    pass
+
+invalid_mailaddr_ids = [i for i in mailaddr_ids if i not in wblist_ids]
+remove_from_one_table(sql_table='mailaddr',
+                      index_column='id',
+                      removed_values=invalid_mailaddr_ids)
+
+logger.info('')
+logger.info('Remained records:')
+logger.info('')
+logger.info('      `msgs`: %-7.d' % len(msgs_mail_ids))
+logger.info('`quarantine`: %-7.d' % (len(quar_mail_ids) - len(invalid_quar_mail_ids)))
+logger.info('     `maddr`: %-7.d' % (len(maddr_ids) - len(invalid_maddr_ids)))
+logger.info('  `mailaddr`: %-7.d' % (len(mailaddr_ids) - len(invalid_mailaddr_ids)))
+
+if counter_msgs \
+        or counter_msgrcpt \
+        or invalid_quar_mail_ids \
+        or invalid_maddr_ids \
+        or invalid_mailaddr_ids:
+    msg = 'Removed records: '
+    msg += '%d in msgs, ' % counter_msgs
+    msg += '%d in msgrcpt, ' % counter_msgrcpt
+    msg += '%d in quarantine, ' % len(invalid_quar_mail_ids)
+    msg += '%d in maddr, ' % len(invalid_maddr_ids)
+    msg += '%d in mailaddr.' % len(invalid_mailaddr_ids)
+
+    ira_tool_lib.log_to_iredadmin(msg, admin='cleanup_amavisd_db', event='cleanup_db')
+    logger.info('Log cleanup status.')

tools/cleanup_db.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Remove old records in iRedAdmin SQL database.
+
+# USAGE:
+#
+#   1: Make sure you have proper values for below two parameters:
+#
+#       IREDADMIN_LOG_KEPT_DAYS = 30
+#
+#      Default values is defined in libs/default_settings.py, you can override
+#      them in settings.py. WARNING: DO NOT MODIFY libs/default_settings.py.
+#
+#   2: Test this script in command line directly, make sure no errors in output
+#      message.
+#
+#       # python cleanup_db.py
+#
+#   3: Setup a daily cron job to execute this script. For example: execute
+#      it daily at 1:30AM.
+#
+#       30  1   *   *   *   python /path/to/cleanup_db.py >/dev/null
+#
+# That's all.
+
+import os
+import sys
+import time
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from tools.ira_tool_lib import debug, logger, sql_dbn, get_db_conn, sql_count_id
+
+web.config.debug = debug
+
+backend = settings.backend
+logger.info('Backend: %s' % backend)
+logger.info('SQL server: %s:%d' % (settings.iredadmin_db_host, int(settings.iredadmin_db_port)))
+
+query_size_limit = 100
+
+conn_iredadmin = get_db_conn('iredadmin')
+
+#
+# iredadmin.log
+#
+_days = settings.IREDADMIN_LOG_KEPT_DAYS
+logger.info('Delete old admin activity log (> %d days)' % _days)
+
+if sql_dbn == 'mysql':
+    sql_where = """timestamp < DATE_SUB(NOW(), INTERVAL %d DAY)""" % _days
+elif sql_dbn == 'postgres':
+    sql_where = """timestamp < CURRENT_TIMESTAMP - INTERVAL '%d DAYS'""" % _days
+else:
+    logger.error('Invalid SQL backend: %s' % sql_dbn)
+    sys.exit()
+
+total_before = sql_count_id(conn_iredadmin, 'log')
+conn_iredadmin.delete('log', where=sql_where)
+total_after = sql_count_id(conn_iredadmin, 'log')
+logger.info('\t- %d removed, %d left.' % (total_before - total_after, total_after))
+
+#
+# iredadmin.domain_ownership
+#
+_days = settings.DOMAIN_OWNERSHIP_EXPIRE_DAYS
+logger.info('Delete old domain ownership verification records (> %d days)' % _days)
+
+total_before = sql_count_id(conn_iredadmin, 'domain_ownership')
+conn_iredadmin.delete('domain_ownership', where="expire > %d" % (_days * 24 * 60 * 60))
+total_after = sql_count_id(conn_iredadmin, 'domain_ownership')
+logger.info('\t- %d removed, %d left.' % (total_before - total_after, total_after))
+
+#
+# iredadmin.newsletter_subunsub_confirms
+#
+now = int(time.time())
+_hours = settings.NEWSLETTER_SUBSCRIPTION_REQUEST_KEEP_HOURS
+logger.info('Delete expired newsletter subscription confirm tokens (> %d hours)' % _hours)
+
+total_before = sql_count_id(conn_iredadmin, 'newsletter_subunsub_confirms')
+_expired = now - (_hours * 60 * 60)
+conn_iredadmin.delete('newsletter_subunsub_confirms', where="expired <= %d" % _expired)
+total_after = sql_count_id(conn_iredadmin, 'newsletter_subunsub_confirms')
+logger.info('\t- %d removed, %d left.' % (total_before - total_after, total_after))

tools/delete_mailboxes.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Delete mailboxes which are scheduled to be removed.
+#
+# Notes: iRedAdmin will store maildir path of removed mail users in SQL table
+#        `iredadmin.deleted_mailboxes` (LDAP backends) or
+#        `vmail.deleted_mailboxes` (SQL backends).
+#
+# Usage: Either run this script manually, or run it with a daily cron job.
+#
+#   # python3 delete_mailboxes.py
+#
+# Available arguments:
+#
+#   * --delete-without-timestamp:
+#
+#       [RISKY] If no timestamp string in maildir path, continue to delete it.
+#
+#       With default iRedMail settings, maildir path will contain a timestamp
+#       like this: <domain.com>/u/s/e/username-2016.08.17.09.53.03/
+#       (2016.08.17.09.53.03 is the timestamp), this way all created maildir
+#       paths are unique, even if you removed the user and recreate it with
+#       same mail address.
+#
+#       Without timestamp in maildir path (e.g. <domain.com>/u/s/e/username/),
+#       if you removed a user and recreate it someday, this user will see old
+#       emails in old mailbox (because maildir path is same as old user's). So
+#       it becomes RISKY to remove the mailbox if no timestamp in maildir path.
+#
+#   * --delete-null-date:
+#
+#       Delete mailbox if SQL column `deleted_mailboxes.delete_date` is null.
+#
+#   * --debug: print additional log
+
+import os
+import sys
+import time
+import logging
+import shutil
+import pwd
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+from libs import iredutils
+from tools import ira_tool_lib
+import settings
+
+web.config.debug = ira_tool_lib.debug
+logger = ira_tool_lib.logger
+
+if '--debug' in sys.argv:
+    logger.setLevel(logging.DEBUG)
+
+# Delete if `deleted_mailboxes.delete_date` is null.
+delete_null_date = False
+if '--delete-null-date' in sys.argv:
+    delete_null_date = True
+
+# Make sure there's a timestamp (yyyy.mm.dd.hh.mm.ss) in maildir path,
+# otherwise it's too risky to remove this mailbox -- because the maildir
+# could be reused by another user after old account was removed.
+#
+#   - Safe to remove: <domain.com>/u/s/e/username-<timestamp>/
+#   - Dangerous to remove: <domain.com>/u/s/e/username/
+delete_without_timestamp = False
+if '--delete-without-timestamp' in sys.argv:
+    delete_without_timestamp = True
+
+
+def delete_record(conn_deleted_mailboxes, rid):
+    try:
+        conn_deleted_mailboxes.delete('deleted_mailboxes',
+                                      vars={'id': rid},
+                                      where='id=$id')
+
+        return True,
+    except Exception as e:
+        return False, repr(e)
+
+
+def delete_mailbox(conn_deleted_mailboxes,
+                   record,
+                   all_maildirs=None):
+    rid = record.id
+    username = str(record.username).lower()
+    timestamp = str(record.timestamp)
+    delete_date = record.delete_date
+
+    maildir = record.maildir
+    maildir = maildir.replace('//', '/')    # Remove duplicate '/'
+
+    if delete_without_timestamp:
+        # Make sure no other mailbox is stored under the maildir.
+        if all_maildirs:
+            if not maildir.endswith('/'):
+                maildir += '/'
+
+            for mdir in all_maildirs:
+                if mdir.startswith(maildir) or (mdir == maildir):
+                    logger.error("<<< ABORT, CRITICAL >>> Trying to remove mailbox ({}) owned by user ({}), but there is another mailbox ({}) stored under this directory. Aborted.".format(maildir, username, mdir))
+                    return False
+    else:
+        _dir = maildir.rstrip('/')
+
+        if len(_dir) <= 21:
+            # Why 21 chars:
+            #   - 20 chars: "-<timestamp>". e.g. "-2014.03.26.15.07.25"
+            #   - username contains at least 1 char
+            logger.error("<<< SKIP >>> Seems no timestamp in maildir path (%s), too risky to remove this mailbox." % maildir)
+            return False
+
+        try:
+            # Extract timestamp string, make sure it's a valid time format.
+            ts = _dir[-19:]
+            time.strptime(ts, '%Y.%m.%d.%H.%M.%S')
+        except Exception as e:
+            logger.debug("<<< WARNING >>> Invalid or missing timestamp in maildir path (%s), skip." % maildir)
+            logger.debug("<<< WARNING >>> Error message: %s." % repr(e))
+            return False
+
+    # check maildir path
+    if os.path.isdir(maildir):
+        # Make sure directory is owned by vmail:vmail
+        _dir_stat = os.stat(maildir)
+        _dir_uid = _dir_stat.st_uid
+
+        # Get uid/gid of vmail user
+        owner = pwd.getpwuid(_dir_uid).pw_name
+        if owner != 'vmail':
+            logger.error('<<< ERROR >> Directory is not owned by `vmail` user: uid -> {}, user -> {}.'.format(_dir_uid, owner))
+            return False
+
+        try:
+            msg = '[{}] {}.'.format(username, maildir)
+            msg += ' Account was deleted at {}.'.format(timestamp)
+            if delete_date:
+                msg += ' Mailbox was scheduled to be removed on {}.'.format(delete_date)
+            else:
+                msg += ' Mailbox was scheduled to be removed as soon as possible.'
+
+            logger.info(msg)
+
+            logger.info("Removing mailbox: {}".format(maildir))
+            # Delete mailbox
+            shutil.rmtree(maildir)
+
+            # Log this deletion.
+            ira_tool_lib.log_to_iredadmin(msg,
+                                          admin='cron_delete_mailboxes',
+                                          username=username,
+                                          event='delete_mailboxes')
+        except Exception as e:
+            logger.error('<<< ERROR >> while deleting mailbox ({} -> {}): {}'.format(username, maildir, repr(e)))
+
+    # Delete record.
+    delete_record(conn_deleted_mailboxes=conn_deleted_mailboxes, rid=rid)
+
+
+# Establish SQL connection.
+try:
+    if settings.backend == 'ldap':
+        conn_deleted_mailboxes = ira_tool_lib.get_db_conn('iredadmin')
+
+        from libs.ldaplib.core import LDAPWrap
+        _wrap = LDAPWrap()
+        conn_vmail = _wrap.conn
+    else:
+        conn_deleted_mailboxes = ira_tool_lib.get_db_conn('vmail')
+        conn_vmail = conn_deleted_mailboxes
+except Exception as e:
+    sys.exit('<<< ERROR >>> Cannot connect to SQL database, aborted. Error: %s' % repr(e))
+
+# Get paths of all maildirs.
+sql_where = 'delete_date <= %s' % web.sqlquote(web.sqlliteral('NOW()'))
+if delete_null_date:
+    sql_where = '(delete_date <= %s) OR (delete_date IS NULL)' % web.sqlquote(web.sqlliteral('NOW()'))
+
+qr_mailboxes = conn_deleted_mailboxes.select('deleted_mailboxes', where=sql_where)
+if not qr_mailboxes:
+    logger.debug('No mailbox is scheduled to be removed.')
+
+    if not delete_null_date:
+        logger.debug("To remove mailboxes without schedule date, please run this script with argument '--delete-null-date'.")
+
+    if not delete_without_timestamp:
+        logger.debug("To remove mailboxes without timesamp in maildir path, please run this script with argument '--delete-without-timestamp'. [WARNING] It's RISKY.")
+
+    sys.exit()
+
+# Get all maildir paths used by active mail users.
+#
+# To delete mailbox without timestamp in maildir path, we must make sure:
+#   - maildir is not used by some active user
+#   - no other mailbox is stored under this maildir path
+#
+# Q: Why query all maildir paths instead of querying SQL/LDAP directly?
+# A:
+#   1. LDAP attribute `homeDirectory` doesn't support `sub` (substring) index.
+#   2. if maildir path contains duplicate '/', the validation will fail (not
+#      equal).
+all_maildirs = []
+if delete_without_timestamp:
+    if settings.backend == 'ldap':
+        _qr = conn_vmail.search_s(settings.ldap_basedn,
+                                  2,     # ldap.SCOPE_SUBTREE
+                                  "(objectClass=mailUser)",
+                                  ['homeDirectory'])
+        for (_dn, _ldif) in _qr:
+            _ldif = iredutils.bytes2str(_ldif)
+            if 'homeDirectory' in _ldif:
+                _dir = _ldif['homeDirectory'][0].lower().replace('//', '/')
+                all_maildirs.append(_dir)
+    elif settings.backend in ['mysql', 'pgsql']:
+        # WARNING: always append '/' in returned maildir path.
+        _qr = conn_vmail.select('mailbox',
+                                what="LOWER(CONCAT(storagebasedirectory, '/', storagenode, '/', maildir, '/')) AS maildir")
+
+        all_maildirs = [str(i.maildir).replace('//', '/') for i in _qr]
+
+for r in list(qr_mailboxes):
+    delete_mailbox(conn_deleted_mailboxes=conn_deleted_mailboxes,
+                   record=r,
+                   all_maildirs=all_maildirs)

tools/delete_sessions.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Delete all records in SQL table "iredadmin.sessions" to force
+#          all admins to re-login.
+
+import os
+import sys
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+from tools import ira_tool_lib
+
+web.config.debug = ira_tool_lib.debug
+logger = ira_tool_lib.logger
+
+conn = ira_tool_lib.get_db_conn('iredadmin')
+
+logger.info('Delete all existing sessions to force all admins to re-login.')
+conn.query('DELETE FROM sessions')

tools/dump_disclaimer.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+
+# Author:   Zhang Huangbin <zhb@iredmail.org>
+# Updated:  2012.07.01
+# Purpose:  Dump disclaimer text from OpenLDAP directory server or SQL servers.
+# Requirements: iRedMail-0.5.0 or later releases
+#
+# Shipped within iRedAdmin-Pro: http://www.iredmail.org/admin_panel.html
+
+# USAGE:
+#
+#   - Make sure you have correct backend related settings in iRedAdmin config
+#     file, settings.ini.
+#
+#   - Test this script in command line directly, make sure no errors in output
+#     message.
+#
+#       # python /path/to/dump_disclaimer.py /etc/postfix/disclaimer/
+#
+#   - Setup a cron job to execute this script daily. For example: execute
+#     this script at 2:01AM every day.
+#
+#       1  2   *   *   *   python /path/to/dump_disclaimer.py /etc/postfix/disclaimer/
+#
+# That's all.
+
+import os
+import sys
+import web
+
+web.config.debug = False
+
+# Directory used to store disclaimer files.
+# Default directory is /etc/postfix/disclaimer/.
+# Default disclaimer file name is [domain_name].txt
+if len(sys.argv) != 2:
+    sys.exit('Error: Please specify a directory used to store disclaimer, default is /etc/postfix/disclaimer/')
+else:
+    DISCLAIMER_DIR = sys.argv[1]
+    DISCLAIMER_FILE_EXT = '.txt'
+
+os.environ['LC_ALL'] = 'C'
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from libs import iredutils
+from tools import ira_tool_lib
+logger = ira_tool_lib.logger
+
+if settings.backend == 'ldap':
+    import ldap
+elif settings.backend == 'mysql':
+    sql_dbn = 'mysql'
+elif settings.backend == 'pgsql':
+    sql_dbn = 'postgres'
+
+
+def write_disclaimer(text, filename, file_type='txt'):
+    # Write plain text
+    try:
+        f = open(filename, 'w')
+
+        if file_type == 'html':
+            html = """<div id="disclaimer_separator"><p>----------</p><br /></div>"""
+            html += """<div id="disclaimer_text"><p>""" + text + """</p></div>"""
+
+            f.write('\n' + html + '\n')
+        else:
+            f.write('\n---------\n' + text + '\n')
+        logger.info("  + %s" % filename)
+        f.close()
+    except Exception as e:
+        logger.info('<<< ERROR >>> %s' % str(e))
+
+
+def handle_disclaimer(domain, disclaimer_text):
+    """Dump or remove disclaimer text."""
+    txt = os.path.join(DISCLAIMER_DIR, domain + '.txt')
+    html = os.path.join(DISCLAIMER_DIR, domain + '.html')
+
+    if disclaimer_text:
+        write_disclaimer(text=disclaimer_text,
+                         filename=txt,
+                         file_type='txt')
+
+        write_disclaimer(text=disclaimer_text,
+                         filename=html,
+                         file_type='html')
+    else:
+        # Remove old disclaimer file if no disclaimer setting
+        try:
+            for f in [txt, html]:
+                if os.path.isfile(f):
+                    os.remove(f)
+                    logger.info("  - Remove %s." % f)
+        except OSError:
+            pass
+        except Exception as e:
+            # Other errors.
+            logger.info("<<< ERROR >>> {}: {}.".format(domain, str(e)))
+
+
+def dump_from_ldap():
+    """Dump disclaimer text from LDAP server."""
+    logger.info('Connecting to LDAP server')
+    conn = ldap.initialize(uri=settings.ldap_uri,
+                           trace_level=0,
+                           bytes_strictness='silent')
+    conn.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
+
+    logger.info('Binding with dn: %s' % settings.ldap_basedn)
+    conn.bind_s(settings.ldap_bind_dn, settings.ldap_bind_password)
+
+    # Search and get disclaimer.
+    logger.info('Searching all domains')
+    qr = conn.search_s(
+        settings.ldap_basedn,
+        ldap.SCOPE_ONELEVEL,
+        '(objectClass=mailDomain)',
+        ['domainName', 'domainAliasName', 'disclaimer'],
+    )
+
+    logger.info('Dumping ...')
+
+    for (_dn, _ldif) in qr:
+        _ldif = iredutils.bytes2str(_ldif)
+
+        # Get domain names.
+        _domains = _ldif['domainName']
+        _alias_domains = _ldif.get('domainAliasName', [])
+        disclaimer_text = _ldif.get('disclaimer', [''])[0]
+
+        domains = _domains + _alias_domains
+
+        for domain in domains:
+            handle_disclaimer(domain, disclaimer_text)
+
+    conn.unbind()
+    logger.info('Connection closed.')
+
+
+def dump_from_sql():
+    """Dump disclaimer text from MySQL or PostgreSQL server."""
+    logger.info("Connecting to SQL server '%s:%d' as user '%s' ..." % (settings.vmail_db_host,
+                                                                       int(settings.vmail_db_port),
+                                                                       settings.vmail_db_user))
+
+    conn = web.database(dbn=sql_dbn,
+                        host=settings.vmail_db_host,
+                        port=int(settings.vmail_db_port),
+                        db=settings.vmail_db_name,
+                        user=settings.vmail_db_user,
+                        pw=settings.vmail_db_password)
+
+    logger.info('Get all alias domains')
+    qr = conn.select('alias_domain', what='alias_domain, target_domain')
+    alias_domains = {}
+    for i in qr:
+        _alias_domain = str(i.alias_domain).lower()
+        _target_domain = str(i.target_domain).lower()
+
+        if _target_domain in alias_domains:
+            alias_domains[_target_domain].append(_alias_domain)
+        else:
+            alias_domains[_target_domain] = [_alias_domain]
+
+    # Search and get disclaimer.
+    logger.info('Get all primary domains')
+    qr = conn.select('domain', what='domain, disclaimer')
+
+    # Dump disclaimer for every domain.
+    logger.info('Dumping...')
+    for r in qr:
+        domain = str(r.domain).lower()
+        disclaimer_text = r.disclaimer
+
+        domains = [domain] + alias_domains.get(domain, [])
+
+        logger.info(domain)
+        for domain in domains:
+            handle_disclaimer(domain, disclaimer_text)
+
+    logger.info('Completed.')
+
+
+if settings.backend == 'ldap':
+    dump_from_ldap()
+elif settings.backend in ['mysql', 'pgsql']:
+    dump_from_sql()

tools/dump_quarantined_mails.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Dump quarantined emails to given directory (specified on command line).
+#
+# Usage:
+#
+#   python dump_quarantined_mail.py /path/to/dir
+
+import os
+import sys
+import time
+import web
+
+output_dir = sys.argv[1]
+if not os.path.isdir(output_dir):
+    sys.exit("Output directory doesn't exist: %s" % output_dir)
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+from tools.ira_tool_lib import debug, get_db_conn
+
+web.config.debug = debug
+
+now = int(time.time())
+conn_amavisd = get_db_conn('amavisd')
+conn_iredadmin = get_db_conn('iredadmin')
+
+# Get last time
+last_time = 0
+try:
+    qr = conn_iredadmin.select('tracking', what='v', where="k='dump_quarantined_mail'", limit=1)
+    if qr:
+        last_time = int(qr[0].v)
+except:
+    pass
+
+# Get value of all `quarantine.mail_id`.
+try:
+    qr = conn_amavisd.select(['msgs', 'quarantine'],
+                             what='msgs.mail_id AS mail_id',
+                             where='msgs.mail_id=quarantine.mail_id AND msgs.time_num >= %d' % last_time,
+                             group='msgs.mail_id')
+except Exception as e:
+    print('<<< ERROR >>> {}'.format(repr(e)))
+    sys.exit()
+
+total = len(qr)
+print("* Found {} quarantined emails in SQL db.".format(total))
+
+counter = 1
+for r in qr:
+    mail_id = str(r.mail_id)
+    try:
+        records = conn_amavisd.select('quarantine',
+                                      what='mail_text',
+                                      where='mail_id = %s' % web.sqlquote(mail_id),
+                                      order='chunk_ind ASC')
+
+        if not records:
+            continue
+
+        # Combine mail_text as RAW mail message.
+        message = ''
+        for i in list(records):
+            for j in i.mail_text:
+                message += j
+
+        # Write message to file
+        try:
+            eml_path = os.path.join(output_dir, 'spam-' + mail_id)
+            print("[{}/{}] Dumping email to file: {}".format(counter, total, eml_path))
+
+            f = open(eml_path, 'w')
+            f.write(message)
+            f.close()
+        except Exception as e:
+            print('<<< ERROR >>> cannot write file {}'.format(repr(e)))
+    except Exception as e:
+        print("<<< ERROR >>> {}".format(repr(e)))
+
+    counter += 1
+
+# Log last time.
+conn_iredadmin.delete('tracking', where="k='dump_quarantined_mail'")
+conn_iredadmin.insert('tracking', k='dump_quarantined_mail', v=now)

tools/export_last_login.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+Query user last login info from (My)SQL database and display it in a more
+readable format (plain text or html).
+
+Note: You need to follow this tutorial to enable last_login plugin in Dovecot:
+      https://docs.iredmail.org/track.user.last.login.html
+
+Usage:
+
+    python3 export_last_login.py                        # in plain text format
+    python3 export_last_login.py html > export.html     # in html format
+"""
+import os
+import sys
+import time
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from tools import ira_tool_lib
+from libs.iredutils import epoch_seconds_to_gmt
+
+web.config.debug = ira_tool_lib.debug
+logger = ira_tool_lib.logger
+
+if settings.backend == 'ldap':
+    conn = ira_tool_lib.get_db_conn('iredadmin')
+else:
+    conn = ira_tool_lib.get_db_conn('vmail')
+
+# Get output format
+try:
+    export_format = sys.argv[1]
+except:
+    export_format = 'text'
+
+try:
+    qr = conn.select('last_login',
+                     order='last_login DESC')
+except Exception as e:
+    sys.exit("Query failed: {}".format(e))
+
+if export_format == 'html':
+    _now = time.strftime('%Y-%d-%m %H:%M:%S')
+
+    html = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html>
+<head>
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+    <style type="text/css">
+        .th_size, .th_date, .td_size, .td_date {{ white-space: nowrap; }}
+        .tr_date {{ background-color: #DDDDDD; }}
+        .text_align_left {{ text-align: left; }}
+    </style>
+</head>
+
+<body>
+    <h3>User Last Login Time ({0})</h3>
+    <table>
+        <thead>
+            <tr>
+                <th>#</th>
+                <th>Email</th>
+                <th>Time (GMT)</th>
+            </tr>
+        </thead>
+        <tbody>
+    """.format(_now)
+
+counter = 1
+for row in qr:
+    username = row.username
+    seconds = row.last_login
+    last_login = epoch_seconds_to_gmt(seconds)
+
+    if export_format == 'html':
+        html += """
+            <tr>
+                <td>{}</td>
+                <td>{}</td>
+                <td>{}</td>
+            </tr>
+        """.format(counter, username, last_login)
+    else:
+        print("{:6} | {:30} | {}".format(counter, username, last_login))
+
+    counter += 1
+
+if export_format == 'html':
+    html += """</tbody></table></body></html>"""
+    print(html)

tools/import_users.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+# Purpose: Read mail accounts from given plain text file (in specified format),
+#          then create them with iRedAdmin-Pro RESTful API interface.
+#
+# Usage:
+#
+#   - Make sure your iRedAdmin-Pro has RESTful API interface enabled by
+#     following our tutorial:
+#     https://docs.iredmail.org/iredadmin-pro.restful.api.html#enable-restful-api
+#
+#   - Generate file /opt/users.list which contains the mail accounts you want
+#     to import, one account per line, with account info stored in few fields:
+#
+#     1: [REQUIRED] user's full email address.
+#     2: [REQUIRED] plain text or password hash which starts with the password
+#                   scheme name. For example, "{SSHA}xxx", "{SSHA512}xxx".
+#     3: [optional] mailbox quota in MB. Must be an integer number.
+#     4: [optional] full display name.
+#     5: [optional] list of mailing list addresses. If not empty, user will be
+#                   assigned to given mailing lists as a member.
+#
+#                   Notes:
+#
+#                   - Multiple addresses must be separated by ":".
+#                   - If mailing list doesn't exist, it will not be created automatically.
+#     6: [optional] employeeid: employee id.
+#
+#     NOTE: the separator "," for ending EMPTY optional fields is not required.
+#
+#     Samples:
+#
+#       user@domain.com, plain_password
+#       user@domain.com, plain_password, 1024, Zhang Huangbin, list1@domain.com:list2@domain.com
+#       user@domain.com, plain_password, , , list1@domain.com:list2@domain.com
+#       user@domain.com, plain_password, 1024, Zhang Huangbin
+#
+#   - Update 3 parameters in this file:
+#
+#       api_endpoint = ''
+#       verify_cert = True
+#       admin = 'postmaster@a.io'
+#       pw = 'password'
+#
+#     - "api_endpoint" is the endpoint of iRedAdmin-Pro RESTful API.
+#     - With "verify_cert = True", a valid ssl cert is required on API
+#       server (https://). If you don't have a valid ssl cert yet, please set
+#       it to False.
+#     - "admin" is the email address of domain admin which has privilege to
+#             manage the email domain which you're going to import users to.
+#     - "pw" is plain password of domain admin.
+#
+#   - Run commands below to create users listed in the "/opt/users.list" file:
+#
+#       python import_users.py /opt/users.list
+
+import os
+import sys
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+# Endpoint of iRedAdmin-Pro RESTful API
+api_endpoint = 'http://127.0.0.1:8080/api'
+
+# Verify SSL cert of API server.
+# If you don't have a valid SSL cert yet, please set it to False.
+verify_cert = True
+
+# Domain admin email address and password
+admin = 'postmaster@a.io'
+pw = 'www'
+
+# Define the order of fields in each line. Fields must be separated by comma.
+#
+#
+# WARNING: For empty optional fields, a comma is still required as placeholder.
+#
+# Samples:
+#
+#   user@domain.com, plain_password, , ,
+#   user@domain.com, plain_password, 1024, Zhang Huangbin, list1@domain.com:list2@domain.com,
+#   user@domain.com, plain_password, , , list1@domain.com:list2@domain.com,
+#   user@domain.com, plain_password, 1024, Zhang Huangbin,,
+#
+field_map = ['mail', 'password', 'quota', 'name', 'groups', 'employeeid']
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+from libs import iredutils
+
+
+def __get(url, data=None):
+    _url = api_endpoint + url
+    r = requests.get(_url, data=data, cookies=cookies, verify=verify_cert)
+    return r.json()
+
+
+def __post(url, data=None):
+    _url = api_endpoint + url
+    r = requests.post(_url, data=data, cookies=cookies, verify=verify_cert)
+    return r.json()
+
+
+def __put(url, data=None):
+    _url = api_endpoint + url
+    r = requests.put(_url, data=data, cookies=cookies, verify=verify_cert)
+    return r.json()
+
+
+def __delete(url, data=None):
+    _url = api_endpoint + url
+    r = requests.delete(_url, data=data, cookies=cookies, verify=verify_cert)
+    return r.json()
+
+
+def usage():
+    pass
+
+
+if len(sys.argv) != 2 or len(sys.argv) > 2:
+    print("Usage: $ python bulk_import.py /path/to/file")
+    usage()
+    sys.exit()
+else:
+    file = sys.argv[1]
+    if not os.path.exists(file):
+        print("<<< ERROR >>> file does not exist: {}".format(file))
+        sys.exit()
+
+#
+# Login
+#
+r = requests.post(api_endpoint + '/login',
+                  data={'username': admin, 'password': pw},
+                  verify=verify_cert)
+
+# Get returned JSON data
+res = r.json()
+if not res['_success']:
+    sys.exit('Login failed')
+
+cookies = r.cookies
+
+# Read user list.
+f = open(file, 'rb')
+
+for line in f.readlines():
+    line = iredutils.bytes2str(line.strip())
+    fields = line.split(',')
+
+    try:
+        d = {}
+        for (k, v) in zip(field_map, fields):
+            d[k] = v
+    except:
+        sys.exit("<<< ERROR >>> line has invalid format:\n{}".format(line))
+
+    # Get user mail address
+    mail = d.pop('mail')
+    mail.lower()
+    if not iredutils.is_email(mail):
+        sys.exit("<<< ERROR >>> line has invalid user email address: {}\nLine: {}".format(mail, line))
+
+    password = d.pop('password')
+    name = d.pop("name", mail.split("@", 1)[0])
+    quota = d.pop("quota", "0")
+
+    # Get mail address(es) of assigned mailing list(s)
+    groups = d.pop('groups', "")
+    groups.lower()
+    groups = [addr.lower().strip() for addr in groups.split(':') if iredutils.is_email(addr)]
+
+    # Create user
+    res = __post('/user/' + mail,
+                 data={'name': name,
+                       'password': password.strip(),
+                       'quota': quota})
+
+    if res['_success']:
+        print("[OK] Created user: {}".format(mail))
+    else:
+        if res['_msg'] == 'ALREADY_EXISTS':
+            print("[SKIP] Account already exists: {}.".format(mail))
+            continue
+        else:
+            sys.exit('<<< ERROR >>> failed to create user: {}'.format(res))
+
+    if password.startswith('{'):
+        res = __put('/user/' + mail,
+                    data={'password_hash': password})
+
+        if res['_success']:
+            print("  |- [OK] Updated user password (hash): {}".format(mail))
+        else:
+            sys.exit('<<< ERROR >>> failed to updated user password (hash): {}, error: {}'.format(mail, res))
+
+    if groups:
+        for group in groups:
+            res = __put('/ml/' + group,
+                        data={'add_subscribers': mail,
+                              'require_confirm': 'no'})
+
+            if res['_success']:
+                print("  |- [OK] Subscribed user to mailing list: {} -> {}".format(mail, group))
+            else:
+                print('<<< WARNING >>> failed to subscribe user to mailing list: {} -> {}, error: {}'.format(mail, group, res))
+
+    employeeid = d.pop("employeeid", "")
+    if employeeid:
+        res = __put('/user/' + mail,
+                    data={'employeeid': employeeid})
+
+        if res['_success']:
+            print("  |- [OK] Updated employeeid: {}".format(mail))
+        else:
+            sys.exit('<<< ERROR >>> failed to updated employeeid: {}, error: {}'.format(mail, res))

tools/ira_tool_lib.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Library used by other scripts under tools/ directory.
+
+import os
+import sys
+import logging
+import web
+
+debug = False
+
+# Set True to print SQL queries.
+web.config.debug = debug
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from libs import iredutils
+
+backend = settings.backend
+if backend in ['ldap', 'mysql']:
+    sql_dbn = 'mysql'
+elif backend in ['pgsql']:
+    sql_dbn = 'postgres'
+else:
+    sys.exit('Error: Unsupported backend (%s).' % backend)
+
+# logging
+logger = logging.getLogger('iredadmin')
+_ch = logging.StreamHandler(sys.stdout)
+_formatter = logging.Formatter('* %(message)s')
+_ch.setFormatter(_formatter)
+logger.addHandler(_ch)
+logger.setLevel(logging.INFO)
+
+
+def get_db_conn(db_name):
+    if backend == 'ldap' and db_name in ['ldap', 'vmail']:
+        logger.error("""Please use code below to get LDAP connection cursor:\n
+
+from libs.ldaplib.core import LDAPWrap\n
+_wrap = LDAPWrap()\n
+conn = _wrap.conn\n""")
+
+        return None
+
+    try:
+        conn = web.database(
+            dbn=sql_dbn,
+            host=settings.__dict__[db_name + '_db_host'],
+            port=int(settings.__dict__[db_name + '_db_port']),
+            db=settings.__dict__[db_name + '_db_name'],
+            user=settings.__dict__[db_name + '_db_user'],
+            pw=settings.__dict__[db_name + '_db_password'],
+        )
+
+        conn.supports_multiple_insert = True
+        return conn
+    except Exception as e:
+        logger.error(e)
+        return None
+
+
+# Log in `iredadmin.log`
+def log_to_iredadmin(msg, event, admin='', username='', loglevel='info'):
+    conn = get_db_conn('iredadmin')
+
+    try:
+        conn.insert('log',
+                    admin=admin,
+                    username=username,
+                    event=event,
+                    loglevel=loglevel,
+                    msg=str(msg),
+                    ip='127.0.0.1',
+                    timestamp=iredutils.get_gmttime())
+    except:
+        pass
+
+    return None
+
+
+def sql_count_id(conn, table, column='id', where=None):
+    if where:
+        qr = conn.select(table,
+                         what='count(%s) as total' % column,
+                         where=where)
+    else:
+        qr = conn.select(table,
+                         what='count(%s) as total' % column)
+    if qr:
+        total = qr[0].total
+    else:
+        total = 0
+
+    return total

tools/migrate_cluebringer_wblist_to_amavisd.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Migrate Cluebringer white/blacklist to Amavisd database.
+#
+# Note: it's safe to execute this script as many times as you want, it won't
+#       generate duplicate records.
+
+import os
+import sys
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from libs.iredutils import is_valid_amavisd_address
+from libs.amavisd import wblist
+from tools import ira_tool_lib
+
+web.config.debug = ira_tool_lib.debug
+logger = ira_tool_lib.logger
+
+# Check database name to make sure it's Cluebringer
+if settings.policyd_db_name != 'cluebringer':
+    sys.exit('Error: not a Cluebringer database.')
+
+logger.info('Establish SQL connection.')
+conn = ira_tool_lib.get_db_conn('policyd')
+
+logger.info('Query white/blacklist info.')
+
+# Converted wblist
+wl = []
+bl = []
+
+# value of sql column: policy_groups.id
+wl_id = None
+bl_id = None
+wb_ids = []
+
+# query whitelist and/or blacklist. possible values: 'wl', 'bl'.
+query_lists = []
+
+# get policy_groups.id
+qr = conn.select('policy_groups', what='id,name', where="name IN ('whitelists', 'blacklists')")
+if qr:
+    for r in qr:
+        if r.name == 'whitelists':
+            wl_id = r.id
+        elif r.name == 'blacklists':
+            bl_id = r.id
+
+    if wl_id:
+        logger.info('policy_groups.id: %d -> whitelists' % wl_id)
+        query_lists.append('wl')
+        wb_ids.append(wl_id)
+
+    if bl_id:
+        logger.info('policy_groups.id: %d -> blacklists' % bl_id)
+        query_lists.append('bl')
+        wb_ids.append(bl_id)
+else:
+    logger.info('No whitelist/blacklist found. Exit.')
+    sys.exit()
+
+logger.info('Query all whitelists and blacklists.')
+qr = conn.select('policy_group_members',
+                 vars={'wb_ids': wb_ids},
+                 what='policygroupid, member',
+                 where='policygroupid IN $wb_ids AND disabled=0')
+
+if qr:
+    logger.info('Convert Cluebringer white/blacklists to Amavisd syntax format.')
+    for r in qr:
+        # Single IP Address: 192.168.2.10
+        # CIDR formatted range of IP addresses: 192.168.2.10/31
+        # Single user: user@example.com
+        # Entire domain: @example.com
+        # All sub-domains: .example.com
+        value = None
+        if is_valid_amavisd_address(r.member):
+            value = r.member
+        else:
+            # Convert from different syntax format
+            if r.member.startswith('.'):
+                tmp = '@' + r.member
+                if is_valid_amavisd_address(tmp):
+                    value = tmp
+                else:
+                    logger.info('[?] Discard record in improper format: %s, cannot convert.' % r.member)
+            elif '/' in r.member:
+                logger.info('[?] Discard record in improper format: %s. CIDR IP range is not supported.' % r.member)
+
+        if value:
+            if r.policygroupid == wl_id:
+                wl.append(value)
+            else:
+                bl.append(value)
+
+if wl:
+    logger.info('Converted whitelisted: %d total' % len(wl))
+else:
+    logger.info('No whitelists found.')
+
+if bl:
+    logger.info('Converted blacklisted: %d total' % len(bl))
+else:
+    logger.info('No blacklists found.')
+
+confirm = input('Migrate converted white/blacklists to Amavisd database right now? [y|N]')
+if confirm not in ['y', 'Y', 'yes', 'YES']:
+    logger.info('Exit without migrating to Amavisd database.')
+    sys.exit()
+
+# Import to Amavisd database.
+try:
+    logger.info('Migrating, please wait ...')
+    wblist.add_wblist(account='@.',
+                      wl_senders=wl,
+                      bl_senders=bl,
+                      flush_before_import=False)
+
+    logger.info("Don't forget to enable iRedAPD plugin 'amavisd_wblist' in /opt/iredapd/settings.py.")
+except Exception as e:
+    logger.info(str(e))
+
+# Ask to delete wblist in cluebringer
+confirm = input('Delete all white/blacklists stored in Cluebringer database? [y|N]')
+if confirm not in ['y', 'Y', 'yes', 'YES']:
+    logger.info('Exit without deleting Cluebringer white/blacklists.')
+    sys.exit()
+
+conn.delete('policy_group_members', vars={'wb_ids': wb_ids}, where='policygroupid IN $wb_ids')
+conn.delete('policy_groups', vars={'wb_ids': wb_ids}, where='id IN $wb_ids')
+conn.delete('policy_members', where="destination='%%internal_domains' AND source IN ('%%whitelists', '%%blacklists')")
+
+# Get policies.id
+qr = conn.select('policies', what='id', where="name IN ('whitelists', 'blacklists')")
+if qr:
+    pids = [r.id for r in qr]
+
+    conn.delete('access_control', vars={'pids': pids}, where='policyid IN $pids')
+    conn.delete('policies', vars={'pids': pids}, where='id IN $pids')
+
+logger.info('DONE')

tools/notify_quarantined_recipients.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html>
+<head>
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+    <style type="text/css">
+        .th_size, .th_date, .td_size, .td_date { white-space: nowrap; }
+        .tr_date { background-color: #DDDDDD; }
+        .text_align_left { text-align: left; }
+    </style>
+</head>
+
+<body>
+
+    <p>Quarantined mails will be kept for %(quar_keep_days)d days, please login to self-service site to manage them: <a href="%(iredadmin_url)s" target="_blank" rel='noopener'>%(iredadmin_url)s</a>.</p>
+
+    <p>Date and time are in time zone: %(timezone)s.</p>
+
+    <table cellpadding="2px">
+        <thead>
+            <th class="th_subject">Subject</th>
+            <th class="th_sender">Sender</th>
+            <th class="th_spam_level">Spam Level</th>
+            <th class="th_date">Time</th>
+        </thead>
+        <tbody>
+            <!-- Info of quarantined mails -->
+            %(quar_mail_info)s
+        </tbody>
+    </table>
+</body>
+</html>

tools/notify_quarantined_recipients.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Notify local recipients (via email) that they have emails
+#          quarantined on server and not delivered to their mailbox.
+
+# Usage:
+#
+#   - Set a correct URL in iRedAdmin-Pro config file `settings.py`, so that
+#     users can manage quarantined email within received notification email:
+#
+#       # URL of your iRedAdmin-Pro login page which will be shown in notification
+#       # email, so that user can login to manage quarantined emails.
+#       # Sample: 'https://your_server.com/iredadmin/'
+#       #
+#       # Note: mail domain must have self-service enabled, otherwise normal
+#       #       mail user cannot login to iRedAdmin-Pro for self-service.
+#       NOTIFICATION_URL_SELF_SERVICE = 'https://[your_server]/iredadmin/'
+#
+#   - Setup a cron job to run this script every 6 or 12, 24 hours, it's up to
+#     you. Sample cron job (every 12 hours):
+#
+#       1 */12 * * * python /path/to/notify_quarantined_recipients.py >/dev/null
+#
+#     Available arguments:
+#
+#       --force-all:
+#           Send notification to all users (who have email quarantined).
+#
+#       --force-all-time:
+#           Notify users for their all quarantined emails instead of just new
+#           ones since last notification.
+#
+#       --notify-backupmx
+#           Send notification to all recipients under backup mx domain
+#
+#   - Also, it's ok to run this script manually:
+#
+#       # python notify_quarantined_recipients.py [arg1 arg2 arg3 ...]
+
+# Customization
+#
+#   - This script sends email via /usr/sbin/sendmail command by default, it
+#     should work quite well and has better performance. if you still prefer
+#     to send notification email via smtp, please set proper smtp server and
+#     account info in iRedAdmin-Pro config file `settings.py`:
+#
+#       NOTIFICATION_SMTP_SERVER = 'localhost'
+#       NOTIFICATION_SMTP_PORT = 587
+#       NOTIFICATION_SMTP_STARTTLS = True
+#       NOTIFICATION_SMTP_USER = ''
+#       NOTIFICATION_SMTP_PASSWORD = ''
+#
+#   - To custom mail subject of notification email, please define below
+#     variable in iRedAdmin-Pro config file `settings.py`:
+#
+#       # Subject of notification email.
+#       NOTIFICATION_QUARANTINE_MAIL_SUBJECT = '[Attention] You have emails quarantined and not delivered to mailbox'
+#
+#   - To custom HTML template file, please create your own template file with
+#     correct name in either place:
+#
+#       - `/opt/iredmail/custom/iredadmin/notify_quarantined_recipients.html`
+#
+#           This file is used if your iRedMail server was deployed with the
+#           iRedMail Easy platform (https://www.iredmail.org/easy.html), easy
+#           for iRedAdmin-Pro upgrade.
+#
+#       - `tools/notify_quarantined_recipients.html.custom` under iRedAdmin-Pro
+#          directory.
+#
+#           General use. Note: there's a `.custom` suffix in file name.
+#
+#     If no custom file, `tools/notify_quarantined_recipients.html` will be used.
+#
+# How it works:
+#
+#   - Mail user login to iRedAdmin-Pro (self-service) and choose to receive
+#     notification email when there's email quarantined.
+#
+#       - OpenLDAP: user will be assigned `enabledService=quar_notify`.
+#       - SQL backends: column `mailbox.settings` contains `quar_notify:yes`.
+#
+#   - This script queries SQL/LDAP database to see who are willing to receive
+#     a notification email.
+#
+#   - This script checks Amavisd database to get info of quarantined mails
+#     for these users.
+
+import os
+import sys
+import time
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.header import Header
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+script_dir = os.path.abspath(os.path.dirname(__file__))
+rootdir = script_dir + '/../'
+sys.path.insert(0, rootdir)
+
+now = int(time.time())
+
+import settings
+from libs import iredutils
+from libs.ireddate import utc_to_timezone
+from tools import ira_tool_lib
+
+web.config.debug = ira_tool_lib.debug
+logger = ira_tool_lib.logger
+
+backend = settings.backend
+
+# Read template HTML file.
+custom_easy_tmpl = "/opt/iredmail/custom/iredadmin/notify_quarantined_recipients.html"
+custom_tmpl = os.path.join(rootdir, 'tools', 'notify_quarantined_recipients.html.custom')
+default_tmpl = os.path.join(rootdir, 'tools', 'notify_quarantined_recipients.html')
+
+if os.path.isfile(custom_easy_tmpl):
+    html_tmpl = custom_easy_tmpl
+elif os.path.isfile(custom_tmpl):
+    html_tmpl = custom_tmpl
+else:
+    html_tmpl = default_tmpl
+
+# Info used in notification email.
+mail_subject = settings.NOTIFICATION_QUARANTINE_MAIL_SUBJECT
+smtp_user = settings.NOTIFICATION_SMTP_USER
+iredadmin_url = settings.NOTIFICATION_URL_SELF_SERVICE
+
+# Use '--force-all' option to notify all mail users.
+force_all_users = '--force-all' in sys.argv or False
+force_all_time = '--force-all-time' in sys.argv or False
+notify_backupmx = '--notify-backupmx' in sys.argv or False
+
+# Backup MX domains.
+# We may not have any accounts under backup mx domain, so if sys admin chooses
+# to notify recipients in backup mx domain, we send the notification also.
+backupmx_domains = []
+
+# List of target users' email address.
+target_users = []
+
+# Get list of users (email) who asked to receive notification email.
+if settings.backend == 'ldap':
+    from libs.ldaplib.core import LDAPWrap
+    _wrap = LDAPWrap()
+    conn_ldap = _wrap.conn
+
+    # Get users who ask to get a notification email under each domain.
+    if force_all_users:
+        q_filter = '(&(objectClass=mailUser)(accountStatus=active))'
+    else:
+        q_filter = '(&(objectClass=mailUser)(accountStatus=active)(enabledService=quar_notify))'
+
+    try:
+        qr = conn_ldap.search_s(settings.ldap_basedn,
+                                2,     # ldap.SCOPE_SUBTREE,
+                                q_filter,
+                                ['mail'])
+        for (_dn, _ldif) in qr:
+            _ldif = iredutils.bytes2str(_ldif)
+            target_users += _ldif.get('mail', [])
+    except Exception as e:
+        logger.info('<< ERROR >> Error while querying mail users: %s' % repr(e))
+
+    if notify_backupmx:
+        # Query all backup mx domains
+        q_filter = '(&(objectClass=mailDomain)(accountStatus=active)(domainBackupMX=yes)(mtaTransport=relay:*))'
+
+        try:
+            qr = conn_ldap.search_s(settings.ldap_basedn,
+                                    1,     # ldap.SCOPE_ONELEVEL,
+                                    q_filter,
+                                    ['domainName', 'domainAliasName'])
+            for (_dn, _ldif) in qr:
+                _ldif = iredutils.bytes2str(_ldif)
+                backupmx_domains += _ldif.get('domainName', [])
+                backupmx_domains += _ldif.get('domainAliasName', [])
+        except Exception as e:
+            logger.info('<< ERROR >> Error while querying backup MX domains: %s' % repr(e))
+
+elif settings.backend in ['mysql', 'pgsql']:
+    conn_vmaildb = ira_tool_lib.get_db_conn('vmail')
+
+    # Get all users who asked to receive notification email.
+    if force_all_users:
+        sql_where = 'active=1'
+    else:
+        sql_where = 'settings LIKE %s AND active=1' % web.sqlquote('%' + 'quar_notify:' + '%')
+
+    qr = conn_vmaildb.select('mailbox',
+                             what='username',
+                             where=sql_where)
+
+    for r in qr:
+        target_users.append(r.username)
+
+    if notify_backupmx:
+        # Get all backup mx domains
+        qr = conn_vmaildb.select('domain',
+                                 what='domain',
+                                 where='backupmx=1 AND active=1')
+        for i in qr:
+            backupmx_domains += [str(i.domain).lower()]
+
+    if backupmx_domains:
+        # Get all alias domains
+        qr = conn_vmaildb.select('alias_domain',
+                                 vars={'domains': backupmx_domains},
+                                 what='alias_domain',
+                                 where='target_domain IN $domains')
+
+        for i in qr:
+            backupmx_domains += [str(i.alias_domain).lower()]
+
+if not (target_users or backupmx_domains):
+    logger.debug('No user asks to receive notification email. Exit.')
+    sys.exit()
+
+mail_body_template = open(html_tmpl).read()
+
+conn_amavisd = ira_tool_lib.get_db_conn('amavisd')
+conn_iredadmin = ira_tool_lib.get_db_conn('iredadmin')
+
+reversed_backupmx_domains = []
+target_backupmx_users = []
+if backupmx_domains:
+    for d in backupmx_domains:
+        rd = d.split('.')
+        rd.reverse()
+        rd = '.'.join(rd)
+
+        reversed_backupmx_domains.append(rd)
+
+    qr = conn_amavisd.select('maddr',
+                             vars={'rcpt': reversed_backupmx_domains},
+                             what='email',
+                             where='domain IN $rcpt')
+    for i in qr:
+        _email = iredutils.bytes2str(i.email)
+        target_backupmx_users.append(_email)
+
+    logger.info('%d backup MX domains (%d users) will receive notification email.' % (len(backupmx_domains), len(target_backupmx_users)))
+
+logger.debug('%d users are willing to receive notification email.' % len(target_users))
+
+target_users += target_backupmx_users
+
+# Notify users.
+for user in target_users:
+    # Get maddr.id of recipient
+    qr = conn_amavisd.select('maddr',
+                             vars={'rcpt': user},
+                             what='id',
+                             where='email=$rcpt',
+                             limit=1)
+    if qr:
+        rid = qr[0].id
+    else:
+        logger.debug('[SKIP] No log of user: ' + user)
+        continue
+
+    # Get info of quarantined mails
+    sql_what = 'msgrcpt.rid AS rid,' \
+               + 'msgs.mail_id AS mail_id,' \
+               + 'msgs.subject AS subject,' \
+               + 'msgs.from_addr AS from_addr,' \
+               + 'msgs.spam_level AS spam_level,' \
+               + 'msgs.time_num'
+
+    sql_where = """msgrcpt.rid=$rid AND msgs.mail_id=msgrcpt.mail_id AND msgs.quar_type='Q'"""
+
+    last_notify_time = 0
+    if not force_all_time:
+        # Get last time
+        try:
+            qr = conn_iredadmin.select('tracking', what='v', where="k='quarantine_notify_time'", limit=1)
+            if qr:
+                last_notify_time = int(qr[0].v) or 0
+        except:
+            pass
+
+    if last_notify_time:
+        sql_where += """ AND msgs.time_num >= %s""" % last_notify_time
+
+    qr = conn_amavisd.select(['msgs', 'msgrcpt'],
+                             vars={'rid': rid},
+                             what=sql_what,
+                             where=sql_where,
+                             order='msgs.time_num DESC')
+
+    if not qr:
+        logger.debug('[SKIP] No quarantined emails for %s.' % user)
+        continue
+
+    total = len(qr)
+
+    # Group messages by date.
+    info_by_date = {}
+
+    quar_mail_info = '\n'
+
+    # Create a HTML table to present quarantined emails.
+    for rcd in qr:
+        # time format: Apr 4, 2015
+        dt = iredutils.epoch_seconds_to_gmt(iredutils.bytes2str(rcd.time_num))
+        time_with_tz = utc_to_timezone(dt=dt, timezone=settings.LOCAL_TIMEZONE)
+        try:
+            time_tuple = time_with_tz.timetuple()
+        except:
+            time_tuple = time.strptime(time_with_tz, '%Y-%m-%d %H:%M:%S')
+
+        mail_date = time.strftime('%b %d, %Y', time_tuple)
+        mail_time = time.strftime('%H:%M:%S', time_tuple)
+
+        info = '<tr>' + '\n'
+        info += '<td class="td_subject">' + iredutils.bytes2str(rcd.subject) + '</td>' + '\n'
+        info += '<td class="td_sender">' + iredutils.bytes2str(rcd.from_addr) + '</td>' + '\n'
+        info += '<td class="td_spam_level">' + iredutils.bytes2str(rcd.spam_level) + '</td>' + '\n'
+        info += '<td class="td_date">' + mail_time + '</td>' + '\n'
+        info += '</tr>' + '\n\n'
+
+        if mail_date not in info_by_date:
+            info_by_date[mail_date] = []
+
+        info_by_date[mail_date].append(info)
+
+    for _date in sorted(list(info_by_date.keys()), reverse=True):
+        quar_mail_info += '<tr class="tr_date"><td colspan="4">' + _date + '</td></tr>' + '\n'
+        for r in info_by_date[_date]:
+            quar_mail_info += r
+
+    msg = MIMEMultipart('alternative')
+
+    msg['Subject'] = Header(mail_subject % {'total': total}, 'utf-8')
+    msg['To'] = user
+
+    if settings.NOTIFICATION_SENDER_NAME:
+        msg['From'] = '{} <{}>'.format(Header(settings.NOTIFICATION_SENDER_NAME, 'utf-8'), smtp_user)
+    else:
+        msg['From'] = Header(smtp_user, 'utf-8')
+
+    mail_body = mail_body_template % {'quar_mail_info': quar_mail_info,
+                                      'quar_keep_days': settings.AMAVISD_REMOVE_QUARANTINED_IN_DAYS,
+                                      'iredadmin_url': iredadmin_url,
+                                      'timezone': settings.LOCAL_TIMEZONE}
+
+    # HTML email must contain text and html part with same content, otherwise
+    # it will be considered as not well-formated email.
+    body_part_plain = MIMEText(mail_body, 'plain', 'utf-8')
+    msg.attach(body_part_plain)
+
+    body_part_html = MIMEText(mail_body, 'html', 'utf-8')
+    msg.attach(body_part_html)
+
+    msg_string = msg.as_string()
+
+    ret = iredutils.sendmail(recipients=user, message_text=msg_string)
+    if ret[0]:
+        logger.info('+ %s: %d mails.' % (user, total))
+    else:
+        logger.info('+ << ERROR >> Error while sending notification email to {}: {}'.format(user, ret[1]))
+
+# Log last notify time.
+conn_iredadmin.delete('tracking', where="k='quarantine_notify_time'")
+conn_iredadmin.insert('tracking', k='quarantine_notify_time', v=now)

tools/promote_user_to_global_admin.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Promote given user to be a global admin.
+#          FYI https://docs.iredmail.org/promote.user.to.be.global.admin.html
+# Usage:
+#   python3 promote_to_global_admin.py <user-email>
+
+
+def usage():
+    print("""Usage: Run this script with user email address:
+
+        # python3 promote_to_global_admin.py user@domain.com
+    """)
+
+
+import os
+import sys
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from tools.ira_tool_lib import debug, get_db_conn
+from libs.iredutils import is_email
+
+backend = settings.backend
+web.config.debug = debug
+
+# Check arguments
+if len(sys.argv) == 2:
+    email = sys.argv[1]
+
+    if not is_email(email):
+        usage()
+        sys.exit()
+else:
+    usage()
+    sys.exit()
+
+if backend == 'ldap':
+    from libs.ldaplib.core import LDAPWrap
+    from libs.ldaplib import ldaputils
+    _wrap = LDAPWrap()
+    conn = _wrap.conn
+
+    dn = ldaputils.rdn_value_to_user_dn(email)
+    mod_attrs = ldaputils.attr_ldif(attr="enabledService", value="domainadmin", mode="add")
+    mod_attrs += ldaputils.attr_ldif(attr="domainGlobalAdmin", value="yes", mode="add")
+
+    try:
+        conn.modify_s(dn, mod_attrs)
+        print("User {} is now a global admin.".format(email))
+    except Exception as e:
+        print("<<< ERROR >>> {}".format(repr(e)))
+
+elif backend in ['mysql', 'pgsql']:
+    conn = get_db_conn('vmail')
+    try:
+        conn.update("mailbox",
+                    isadmin=1,
+                    isglobaladmin=1,
+                    where="username='{}'".format(email))
+
+        conn.insert("domain_admins",
+                    username=email,
+                    domain="ALL")
+
+        print("User {} is now a global admin.".format(email))
+    except Exception as e:
+        print("<<< ERROR >>> {}".format(repr(e)))

tools/reset_user_password.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Update user password.
+# Usage:
+#   python reset_user_password.py <email> <new_password>
+
+
+def usage():
+    print("""Usage: Run this script with user email address and new plain password:
+
+        # python3 reset_user_password.py user@domain.com 123456
+    """)
+
+
+import os
+import sys
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from tools.ira_tool_lib import debug, get_db_conn
+from libs.iredutils import is_email
+from libs.iredpwd import generate_password_hash
+
+backend = settings.backend
+web.config.debug = debug
+
+# Check arguments
+if len(sys.argv) == 3:
+    email = sys.argv[1]
+    pw = sys.argv[2]
+
+    if not is_email(email):
+        usage()
+        sys.exit()
+else:
+    usage()
+    sys.exit()
+
+pw_hash = generate_password_hash(pw)
+if backend == 'ldap':
+    from libs.ldaplib.core import LDAPWrap
+    from libs.ldaplib import ldaputils
+    _wrap = LDAPWrap()
+    conn = _wrap.conn
+
+    dn = ldaputils.rdn_value_to_user_dn(email)
+    mod_attrs = ldaputils.mod_replace('userPassword', pw_hash)
+    mod_attrs += ldaputils.mod_replace('shadowLastChange', ldaputils.get_days_of_shadow_last_change())
+
+    try:
+        conn.modify_s(dn, mod_attrs)
+        print("[{}] Password has been reset.".format(email))
+    except Exception as e:
+        print("<<< ERROR >>> {}".format(repr(e)))
+elif backend in ['mysql', 'pgsql']:
+    conn = get_db_conn('vmail')
+    try:
+        conn.update('mailbox',
+                    password=pw_hash,
+                    passwordlastchange=web.sqlliteral('NOW()'),
+                    where="username='{}'".format(email))
+
+        print("[{}] Password has been reset.".format(email))
+    except Exception as e:
+        print("<<< ERROR >>> {}".format(repr(e)))

tools/update_mailbox_quota.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Update mailbox quota for one user or multiple users.
+# Note: Mailbox quota size unit is bytes. for example, size `104857600` is 100 MB.
+
+
+def usage():
+    print("""Usage:
+
+    1) Update mailbox quota for one user.
+
+       To simply update one user's quota, run this script with user's email
+       address and new quota size (in bytes). For example:
+
+        # python3 update_mailbox_quota.py user@domain.com 2048576000
+
+    2) Update mailbox quota for multiple users.
+
+       - Create text file "new_quota.txt", each line contains one email address
+         and the new quota size (in bytes).
+
+            user1@domain.com 20480000
+            user2@domain.com 102400000
+            user3@domain.com 409600000
+
+       - Run this script with this file:
+
+            # python3 update_mailbox_quota.py new_quota.txt
+    """)
+
+
+import os
+import sys
+import web
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from tools.ira_tool_lib import debug, logger, get_db_conn
+from libs.iredutils import is_email
+
+backend = settings.backend
+logger.info('Backend: {}'.format(backend))
+
+web.config.debug = debug
+
+# List of (email, quota) tuples.
+users = []
+
+# Check arguments
+if len(sys.argv) == 2:
+    # bulk update
+    text_file = sys.argv[1]
+    if not os.path.isfile(text_file):
+        sys.exit('<<< ERROR>>> Not a regular file: %s' % text_file)
+
+    # Get all (email, quota) tuples.
+    f = open(text_file)
+    for _line in f.readlines():
+        (_email, _quota) = _line.strip().split(' ', 1)
+        if is_email(_email) and _quota.isdigit():
+            users += [(_email, _quota)]
+        else:
+            print("[SKIP] no valid email address or quota: {}".format(_line))
+
+elif len(sys.argv) == 3:
+    # update single user
+    _email = sys.argv[1]
+    _quota = sys.argv[2]
+
+    if is_email(_email):
+        users += [(_email, _quota)]
+    else:
+        sys.exit('<<< ERROR >>> Not an valid email address: %s' % _email)
+else:
+    usage()
+
+total = len(users)
+logger.info('{} users in total.'.format(total))
+
+count = 1
+if backend == 'ldap':
+    from libs.ldaplib.core import LDAPWrap
+    from libs.ldaplib.ldaputils import rdn_value_to_user_dn, mod_replace
+    _wrap = LDAPWrap()
+    conn = _wrap.conn
+
+    for (_email, _quota) in users:
+        logger.info('(%d/%d) Updating %s -> %s' % (count, total, _email, _quota))
+        dn = rdn_value_to_user_dn(_email)
+        mod_attrs = mod_replace('mailQuota', _quota)
+        try:
+            conn.modify_s(dn, mod_attrs)
+        except Exception as e:
+            print("<<< ERROR >>> {}".format(e))
+elif backend in ['mysql', 'pgsql']:
+    conn = get_db_conn('vmail')
+    for (_email, _quota) in users:
+        logger.info('(%d/%d) Updating %s -> %s' % (count, total, _email, _quota))
+        conn.update('mailbox',
+                    quota=int(_quota),
+                    where="username='%s'" % _email)

tools/update_password_in_csv.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+# Author: Zhang Huangbin <zhb@iredmail.org>
+# Purpose: Update user passwords from records in a CSV file.
+
+import os
+import sys
+import web
+
+
+def usage():
+    print("""Usage:
+
+    - Store the email address and new password in a plain text file, e.g.
+      'passwords.csv'. format is:
+
+          <email> <new_password>
+
+      Samples:
+
+        user1@domain.com pF4mTq4jaRzDLlWl
+        user2@domain.com SPhkTUlZs1TBxvmJ
+        user3@domain.com 8deNR8IBLycRujDN
+
+   - Run this script with this file:
+
+        python3 update_password_in_csv.py passwords.csv
+    """)
+
+
+os.environ['LC_ALL'] = 'C'
+
+rootdir = os.path.abspath(os.path.dirname(__file__)) + '/../'
+sys.path.insert(0, rootdir)
+
+import settings
+from tools.ira_tool_lib import debug, logger, get_db_conn
+from libs.iredutils import is_email
+from libs.iredpwd import generate_password_hash
+
+backend = settings.backend
+logger.info('Backend: %s' % backend)
+
+web.config.debug = debug
+
+logger.info('Parsing command line arguments.')
+
+# File which stores email and quota.
+text_file = ''
+
+# The separator
+column_separator = ' '
+
+# List of (email, quota) tuples.
+users = []
+
+# Check arguments
+if len(sys.argv) == 2:
+    text_file = sys.argv[1]
+    if not os.path.isfile(text_file):
+        sys.exit('<<< ERROR>>> Not a regular file: %s' % text_file)
+
+    # Get all (email, password) tuples.
+    f = open(text_file)
+    line_num = 0
+    for _line in f.readlines():
+        line_num += 1
+        (_email, _pw) = _line.split(column_separator, 1)
+        if is_email(_email):
+            users += [(_email, _pw)]
+        else:
+            print("[SKIP] line {}: no valid email address: {}".format(line_num, _line))
+    f.close()
+else:
+    usage()
+
+total = len(users)
+logger.info('%d users in total.' % total)
+
+count = 1
+if backend == 'ldap':
+    from libs.ldaplib.core import LDAPWrap
+    from libs.ldaplib.ldaputils import rdn_value_to_user_dn, mod_replace
+    _wrap = LDAPWrap()
+    conn = _wrap.conn
+
+    for (_email, _pw) in users:
+        logger.info('(%d/%d) Updating %s' % (count, total, _email))
+
+        dn = rdn_value_to_user_dn(_email)
+        pw_hash = generate_password_hash(_pw)
+        mod_attrs = mod_replace('userPassword', pw_hash)
+        try:
+            conn.modify_s(dn, mod_attrs)
+        except Exception as e:
+            print("<<< ERROR >>> {}".format(repr(e)))
+elif backend in ['mysql', 'pgsql']:
+    conn = get_db_conn('vmail')
+    for (_email, _pw) in users:
+        logger.info('(%d/%d) Updating %s' % (count, total, _email))
+        pw_hash = generate_password_hash(_pw)
+        conn.update('mailbox',
+                    password=pw_hash,
+                    where="username='%s'" % _email)

tools/upgrade_iredadmin.sh

@@ -0,0 +1,963 @@
+#!/usr/bin/env bash
+
+# Purpose: Upgrade iRedAdmin from old release.
+#          Works with both iRedAdmin open source edition or iRedAdmin-Pro.
+
+# USAGE:
+#
+#   # cd /path/to/iRedAdmin-xxx/tools/
+#   # bash upgrade_iredadmin.sh
+#
+# Notes:
+#
+#   * it uses sql username 'root' by default to connect to sql database. If you
+#     are using a remote SQL database which you don't have root privilege,
+#     please specify the sql username on command line with 'SQL_IREDADMIN_USER'
+#     parameter like this:
+#
+#       SQL_IREDADMIN_USER='iredadmin' bash upgrade_iredadmin.sh
+#
+#   * it reads sql password for given sql user from /root/.my.cnf by default.
+#     if you use a different file, please specify the file on command line with
+#     'MY_CNF' parameter like this:
+#
+#       MY_CNF='/root/.my.cnf-iredadmin' SQL_IREDADMIN_USER='iredadmin' bash upgrade_iredadmin.sh
+
+export LC_ALL='C'
+export IRA_HTTPD_USER='iredadmin'
+export IRA_HTTPD_GROUP='iredadmin'
+
+export SYS_ROOT_USER='root'
+
+# If you don't have root privilege, use another sql user instead.
+export SQL_IREDADMIN_USER="${SQL_IREDADMIN_USER:=root}"
+export MY_CNF="${MY_CNF:=/root/.my.cnf}"
+export CMD_MYSQL="mysql --defaults-file=${MY_CNF} -u ${SQL_IREDADMIN_USER}"
+
+# Check OS to detect some necessary info.
+export KERNEL_NAME="$(uname -s | tr '[a-z]' '[A-Z]')"
+
+export NGINX_PID_FILE='/var/run/nginx.pid'
+export NGINX_SNIPPET_CONF='/etc/nginx/templates/iredadmin.tmpl'
+export NGINX_SNIPPET_CONF2='/etc/nginx/templates/iredadmin-subdomain.tmpl'
+# iRedMail-0.9.7
+export NGINX_SNIPPET_CONF3='/etc/nginx/conf.d/default.conf'
+
+export USE_SYSTEMD='NO'
+if which systemctl &>/dev/null; then
+    export USE_SYSTEMD='YES'
+    export SYSTEMD_SERVICE_DIR='/lib/systemd/system'
+    export SYSTEMD_SERVICE_DIR2='/etc/systemd/system'
+    export SYSTEMD_SERVICE_USER_DIR='/etc/systemd/system/multi-user.target.wants/'
+fi
+
+# Python.
+export CMD_PYTHON3='/usr/bin/python3'
+export CMD_PIP3='/usr/bin/pip3'
+
+# uwsgi
+export CMD_UWSGI='/usr/bin/uwsgi'
+
+# If uwsgi is installed with pip, plugins are compiled into core binary
+# directly, not plugins are installed separately.
+# Mainly used on RHEL/CentOS/Rocky/Alma.
+export UWSGI_HAS_PLUGINS="YES"
+
+if [ X"${KERNEL_NAME}" == X"LINUX" ]; then
+    # Note: RHEL has minor version number in VERSION_ID.
+    export DISTRO_VERSION=$(awk -F'"' '/^VERSION_ID=/ {print $2}' /etc/os-release | awk -F'.' '{print $1}')
+
+    if [ -f /etc/redhat-release ]; then
+        # RHEL/CentOS
+        export DISTRO='RHEL'
+
+        # Installed with pip.
+        export CMD_UWSGI='/usr/sbin/uwsgi'
+
+        if [[ -x "/usr/local/bin/uwsgi" ]]; then
+            export CMD_UWSGI='/usr/local/bin/uwsgi'
+            export UWSGI_HAS_PLUGINS="NO"
+        fi
+
+        export HTTPD_RC_SCRIPT_NAME='httpd'
+        export CRON_SPOOL_DIR='/var/spool/cron'
+
+        if [[ -L /opt/www/iredadmin ]]; then
+            export HTTPD_SERVERROOT='/opt/www'
+        else
+            export HTTPD_SERVERROOT='/var/www'
+        fi
+    elif [ -f /etc/lsb-release ]; then
+        # Ubuntu
+        export DISTRO='UBUNTU'
+
+        export HTTPD_RC_SCRIPT_NAME='apache2'
+        export CRON_SPOOL_DIR='/var/spool/cron/crontabs'
+
+        if [ -L /opt/www/iredadmin ]; then
+            export HTTPD_SERVERROOT='/opt/www'
+        else
+            export HTTPD_SERVERROOT='/usr/share/apache2'
+        fi
+    elif [ -f /etc/debian_version ]; then
+        # Debian
+        export DISTRO='DEBIAN'
+
+        export HTTPD_RC_SCRIPT_NAME='apache2'
+        export CRON_SPOOL_DIR='/var/spool/cron/crontabs'
+
+        if [ -L /opt/www/iredadmin ]; then
+            export HTTPD_SERVERROOT='/opt/www'
+        else
+            export HTTPD_SERVERROOT='/usr/share/apache2'
+        fi
+    elif [ -f /etc/SuSE-release ]; then
+        # openSUSE
+        export DISTRO='SUSE'
+        export HTTPD_SERVERROOT='/srv/www'
+        export HTTPD_RC_SCRIPT_NAME='apache2'
+        export CRON_SPOOL_DIR='/var/spool/cron'
+    else
+        echo "<<< ERROR >>> Cannot detect Linux distribution name. Exit."
+        echo "Please contact support@iredmail.org to solve it."
+        exit 255
+    fi
+elif [ X"${KERNEL_NAME}" == X'FREEBSD' ]; then
+    export DISTRO='FREEBSD'
+    export SYSRC='/usr/sbin/sysrc'
+
+    export CMD_PYTHON3='/usr/local/bin/python3'
+    export CMD_UWSGI='/usr/local/bin/uwsgi'
+
+    [ -x /usr/local/bin/pip-3.8 ] && export CMD_PIP3='/usr/local/bin/pip-3.8'
+    [ -x /usr/local/bin/pip3 ] && export CMD_PIP3='/usr/local/bin/pip3'
+    [ -x /usr/local/bin/pip ] && export CMD_PIP3='/usr/local/bin/pip'
+
+    export CRON_SPOOL_DIR='/var/cron/tabs'
+    export NGINX_SNIPPET_CONF='/usr/local/etc/nginx/templates/iredadmin.tmpl'
+    export NGINX_SNIPPET_CONF2='/usr/local/etc/nginx/templates/iredadmin-subdomain.tmpl'
+    export NGINX_SNIPPET_CONF3='/usr/local/etc/nginx/conf.d/default.conf'
+
+    if [ -L /opt/www/iredadmin ]; then
+        export HTTPD_SERVERROOT='/opt/www'
+    else
+        export HTTPD_SERVERROOT='/usr/local/www'
+    fi
+
+    if [ -f /usr/local/etc/rc.d/apache24 ]; then
+        export HTTPD_RC_SCRIPT_NAME='apache24'
+    else
+        export HTTPD_RC_SCRIPT_NAME='apache22'
+    fi
+elif [ X"${KERNEL_NAME}" == X'OPENBSD' ]; then
+    export CMD_PYTHON3='/usr/local/bin/python3'
+    export CMD_PIP3='/usr/local/bin/pip3'
+    export CMD_UWSGI='/usr/local/bin/uwsgi'
+    export DISTRO='OPENBSD'
+    export CRON_SPOOL_DIR='/var/cron/tabs'
+
+    if [[ -h /opt/www/iredadmin ]]; then
+        export HTTPD_SERVERROOT='/opt/www'
+    else
+        export HTTPD_SERVERROOT='/var/www'
+    fi
+else
+    echo "Cannot detect Linux/BSD distribution. Exit."
+    echo "Please contact author iRedMail team <support@iredmail.org> to solve it."
+    exit 255
+fi
+
+export CRON_FILE_ROOT="${CRON_SPOOL_DIR}/${SYS_ROOT_USER}"
+
+# Optional argument to set the directory which stores iRedAdmin.
+if [ $# -gt 0 ]; then
+    if [ -d ${1} ]; then
+        export HTTPD_SERVERROOT="${1}"
+    fi
+
+    if echo ${HTTPD_SERVERROOT} | grep '/iredadmin/*$' > /dev/null; then
+        export HTTPD_SERVERROOT="$(dirname ${HTTPD_SERVERROOT})"
+    fi
+fi
+
+# iRedAdmin directory and config file.
+export IRA_ROOT_DIR="${HTTPD_SERVERROOT}/iredadmin"
+export IRA_CONF_PY="${IRA_ROOT_DIR}/settings.py"
+export IRA_CUSTOM_CONF_PY="${IRA_ROOT_DIR}/custom_settings.py"
+
+enable_service() {
+    srv="$1"
+
+    echo "* Enable service: ${srv}"
+    if [ X"${DISTRO}" == X'RHEL' ]; then
+        if [ X"${USE_SYSTEMD}" == X'YES' ]; then
+            systemctl enable $srv
+        else
+            chkconfig --level 345 $srv on
+        fi
+    elif [ X"${DISTRO}" == X'DEBIAN' -o X"${DISTRO}" == X'UBUNTU' ]; then
+        if [ X"${USE_SYSTEMD}" == X'YES' ]; then
+            systemctl enable $srv
+        else
+            update-rc.d $srv defaults
+        fi
+    elif [ X"${DISTRO}" == X'FREEBSD' ]; then
+        ${SYSRC} -f /etc/rc.conf.local ${srv}_enable=YES
+    elif [ X"${DISTRO}" == X'OPENBSD' ]; then
+        rcctl enable $srv
+    fi
+}
+
+restart_service() {
+    srv="$1"
+
+    if [ X"${KERNEL_NAME}" == X'LINUX' ]; then
+        if [ X"${USE_SYSTEMD}" == X'YES' ]; then
+            systemctl restart ${srv}
+        else
+            service ${srv} restart
+        fi
+    elif [ X"${KERNEL_NAME}" == X'FREEBSD' ]; then
+        service ${srv} restart
+    elif [ X"${KERNEL_NAME}" == X'OPENBSD' ]; then
+        rcctl restart ${srv}
+    fi
+
+    if [ X"$?" != X'0' ]; then
+        echo "Failed, please restart service manually and check its log file."
+    fi
+}
+
+restart_web_service()
+{
+    export web_service="${HTTPD_RC_SCRIPT_NAME}"
+    if [ -f ${NGINX_PID_FILE} ]; then
+        if [ -n "$(cat ${NGINX_PID_FILE})" ]; then
+            export web_service="iredadmin"
+        fi
+    fi
+
+    echo "* Restarting ${web_service} service."
+    if [ X"${KERNEL_NAME}" == X'LINUX' ]; then
+        # The uwsgi script on CentOS 6 has problem with 'restart' action,
+        # 'stop' with few seconds sleep fixes it.
+        if [ X"${DISTRO}" == X'RHEL' -a X"${web_service}" == X'uwsgi' ]; then
+            service ${web_service} stop
+            sleep 5
+            service ${web_service} start
+        else
+            service ${web_service} restart
+        fi
+    elif [ X"${KERNEL_NAME}" == X'FREEBSD' ]; then
+        service ${web_service} restart
+    elif [ X"${KERNEL_NAME}" == X'OPENBSD' ]; then
+        rcctl restart ${web_service}
+    fi
+
+    if [ X"$?" != X'0' ]; then
+        echo "Failed, please restart Apache web server or 'iredadmin' (if you're running Nginx as web server) manually."
+    fi
+}
+
+check_mlmmjadmin_installation()
+{
+    if [ ! -e /opt/mlmmjadmin ]; then
+        echo "<<< ERROR >>> No mlmmjadmin installation found (/opt/mlmmjadmin)."
+        echo "<<< ERROR >>> Please follow iRedMail upgrade tutorials to the latest"
+        echo "<<< ERROR >>> stable release first, then come back to upgrade iRedAdmin-Pro."
+        echo "<<< ERROR >>> mlmmj and mlmmjadmin was first introduced in iRedMail-0.9.8."
+        echo "<<< ERROR >>> https://docs.iredmail.org/iredmail.releases.html"
+        exit 255
+    fi
+}
+
+remove_pkg() {
+    echo "Remove package(s): $@"
+    if [ X"${DISTRO}" == X'RHEL' ]; then
+        yum remove -y $@
+    fi
+}
+
+install_pkg()
+{
+    echo "Install package(s): $@"
+
+    if [ X"${DISTRO}" == X'RHEL' ]; then
+        yum -y install $@
+    elif [ X"${DISTRO}" == X'DEBIAN' -o X"${DISTRO}" == X'UBUNTU' ]; then
+        apt-get install -y $@
+    elif [ X"${DISTRO}" == X'FREEBSD' ]; then
+        cd /usr/ports/$@ && make install clean
+    elif [ X"${DISTRO}" == X'OPENBSD' ]; then
+        pkg_add -r $@
+    else
+        echo "<< ERROR >> Please install package(s) manually: $@"
+    fi
+}
+
+has_python_module()
+{
+    mod="$1"
+    ${CMD_PYTHON3} -c "import $mod" &>/dev/null
+    if [ X"$?" == X'0' ]; then
+        echo 'YES'
+    else
+        echo 'NO'
+    fi
+}
+
+add_missing_parameter()
+{
+    # Usage: add_missing_parameter VARIABLE DEFAULT_VALUE [COMMENT]
+    var="${1}"
+    value="${2}"
+    shift 2
+    comment="$@"
+
+    if ! grep "^${var}" ${IRA_CONF_PY} &>/dev/null; then
+        if [ ! -z "${comment}" ]; then
+            echo "# ${comment}" >> ${IRA_CONF_PY}
+        fi
+
+        if [ X"${value}" == X'True' -o X"${value}" == X'False' ]; then
+            echo "${var} = ${value}" >> ${IRA_CONF_PY}
+        else
+            # Value must be quoted as string.
+            echo "${var} = '${value}'" >> ${IRA_CONF_PY}
+        fi
+    fi
+}
+
+# Remove all single quote and double quotes in string.
+strip_quotes()
+{
+    # Read input from stdin
+    str="$(cat <&0)"
+    value="$(echo ${str} | tr -d '"' | tr -d "'")"
+    echo "${value}"
+}
+
+get_iredadmin_setting()
+{
+    var="${1}"
+    value="$(grep "^${var}" ${IRA_CONF_PY} | awk '{print $NF}' | strip_quotes)"
+    echo "${value}"
+}
+
+check_dot_my_cnf()
+{
+    if egrep '^backend.*(mysql|ldap)' ${IRA_CONF_PY} &>/dev/null; then
+        if [ ! -f ${MY_CNF} ]; then
+            echo "<<< ERROR >>> File ${MY_CNF} not found."
+            echo "<<< ERROR >>> Please add mysql root user and password in it like below, then run this script again."
+            cat <<EOF
+
+[client]
+host=127.0.0.1
+port=3306
+user=${SQL_IREDADMIN_USER}
+password="plain_password"
+
+EOF
+
+            exit 255
+        fi
+
+        # Check MySQL connection
+        ${CMD_MYSQL} -e "SHOW DATABASES" &>/dev/null
+        if [ X"$?" != X'0' ]; then
+            echo "<<< ERROR >>> MySQL user name '${SQL_IREDADMIN_USER}' or password is incorrect in ${MY_CNF}, please double check."
+            exit 255
+        fi
+    fi
+}
+
+check_mlmmjadmin_installation
+check_dot_my_cnf
+
+echo "* Detected Linux/BSD distribution: ${DISTRO}"
+echo "* HTTP server root: ${HTTPD_SERVERROOT}"
+
+if [ -L ${IRA_ROOT_DIR} ]; then
+    export IRA_ROOT_REAL_DIR="$(readlink ${IRA_ROOT_DIR})"
+    echo "* Found iRedAdmin directory: ${IRA_ROOT_DIR}, symbol link of ${IRA_ROOT_REAL_DIR}"
+else
+    echo "<<< ERROR >>> Directory (${IRA_ROOT_DIR}) is not a symbol link created by iRedMail. Exit."
+    exit 255
+fi
+
+# Copy config file
+if [ -f ${IRA_CONF_PY} ]; then
+    echo "* Found iRedAdmin config file: ${IRA_CONF_PY}"
+else
+    echo "<<< ERROR >>> Cannot find a valid config file (settings.py)."
+    exit 255
+fi
+
+# Check whether current directory is iRedAdmin
+PWD="$(pwd)"
+if ! echo ${PWD} | grep 'iRedAdmin-.*/tools' >/dev/null; then
+    echo "<<< ERROR >>> Cannot find new version of iRedAdmin in current directory. Exit."
+    exit 255
+fi
+
+# Copy current directory to Apache server root
+dir_new_version="$(dirname ${PWD})"
+name_new_version="$(basename ${dir_new_version})"
+NEW_IRA_ROOT_DIR="${HTTPD_SERVERROOT}/${name_new_version}"
+if [ -d ${NEW_IRA_ROOT_DIR} ]; then
+    COPY_FILES="${dir_new_version}/*"
+    COPY_DEST_DIR="${NEW_IRA_ROOT_DIR}"
+    #echo "<<< ERROR >>> Directory exist: ${NEW_IRA_ROOT_DIR}. Exit."
+    #exit 255
+else
+    COPY_FILES="${dir_new_version}"
+    COPY_DEST_DIR="${HTTPD_SERVERROOT}"
+fi
+
+echo "* Copying new version to ${NEW_IRA_ROOT_DIR}"
+cp -rf ${COPY_FILES} ${COPY_DEST_DIR}
+
+# Copy old config files
+echo "* Copy ${IRA_CONF_PY}."
+cp -p ${IRA_CONF_PY} ${NEW_IRA_ROOT_DIR}/
+
+if [ -f ${IRA_CUSTOM_CONF_PY} ]; then
+    echo "* Copy ${IRA_CUSTOM_CONF_PY}."
+    cp -p ${IRA_CUSTOM_CONF_PY} ${NEW_IRA_ROOT_DIR}
+fi
+
+# Copy hooks.py. It's ok if missing.
+if [ -f ${IRA_ROOT_DIR}/hooks.py ]; then
+    echo "* Copy ${IRA_ROOT_DIR}/hooks.py."
+    cp -p ${IRA_ROOT_DIR}/hooks.py ${NEW_IRA_ROOT_DIR}/ &>/dev/null
+fi
+
+# Copy custom files under 'tools/'. It's ok if missing.
+cp -p ${IRA_ROOT_DIR}/tools/*.custom ${NEW_IRA_ROOT_DIR}/tools/ &>/dev/null
+cp -p ${IRA_ROOT_DIR}/tools/*.last-time ${NEW_IRA_ROOT_DIR}/tools/ &>/dev/null
+
+# Template file renamed
+if [ -f "${IRA_ROOT_DIR}/tools/notify_quarantined_recipients.custom.html" ]; then
+    echo "* Copy ${IRA_ROOT_DIR}/tools/notify_quarantined_recipients.custom.html"
+    cp -f ${IRA_ROOT_DIR}/tools/notify_quarantined_recipients.custom.html \
+        ${NEW_IRA_ROOT_DIR}/tools/notify_quarantined_recipients.html.custom
+fi
+
+# Copy favicon.ico and brand logo image.
+for var in 'BRAND_FAVICON' 'BRAND_LOGO'; do
+    if grep "^${var}\>" ${IRA_CONF_PY} &>/dev/null; then
+        _file=$(grep "^${var}\>" ${IRA_CONF_PY} | awk '{print $NF}' | tr -d '"' | tr -d "'")
+        echo "* Copy file ${IRA_ROOT_DIR}/static/${_file}"
+        cp -f ${IRA_ROOT_DIR}/static/${_file} ${NEW_IRA_ROOT_DIR}/static/
+    fi
+done
+
+# iredadmin is now ran as a standalone uwsgi instance, we don't need uwsgi
+# daemon service anymore.
+_uwsgi_confs='
+    /etc/uwsgi.d/iredadmin.ini
+    /etc/uwsgi-available/iredadmin.ini
+    /etc/uwsgi/apps-enabled/iredadmin.ini &>/dev/null
+    /etc/uwsgi/apps-available/iredadmin.ini &>/dev/null
+    /usr/local/etc/uwsgi/iredadmin.ini
+    /etc/uwsgi-enabled/iredadmin.ini &>/dev/null
+    /etc/uwsgi-available/iredadmin.ini &>/dev/null
+'
+
+for f in ${_uwsgi_confs}; do
+    rm -f ${f} &>/dev/null
+done
+
+# Remove 'uwsgi_XXX' from /etc/rc.conf on FreeBSD.
+if [[ X"${DISTRO}" == X'FREEBSD' ]]; then
+    ${SYSRC} -x uwsgi_enable &>/dev/null
+    ${SYSRC} -x uwsgi_profiles &>/dev/null
+    ${SYSRC} -x uwsgi_iredadmin_flags &>/dev/null
+fi
+
+# Update Nginx template file
+export _restart_nginx='NO'
+for f in ${NGINX_SNIPPET_CONF} ${NGINX_SNIPPET_CONF2} ${NGINX_SNIPPET_CONF3}; do
+    if [[ -f ${f} ]]; then
+        if grep 'unix:.*iredadmin.socket' ${f} &>/dev/null; then
+            export _restart_nginx='YES'
+            perl -pi -e 's#uwsgi_pass unix:.*iredadmin.socket;#uwsgi_pass 127.0.0.1:7791;#g' ${f}
+        fi
+    fi
+done
+
+if [[ X"${_restart_nginx}" == X'YES' ]]; then
+    restart_service nginx
+fi
+
+# Update uwsgi ini config file
+if [ -d ${NEW_IRA_ROOT_DIR}/rc_scripts/uwsgi ]; then
+    perl -pi -e 's#^chdir = (.*)#chdir = $ENV{HTTPD_SERVERROOT}/iredadmin#g' ${NEW_IRA_ROOT_DIR}/rc_scripts/uwsgi/*.ini
+fi
+
+# Copy rc script or systemd service file
+if [ X"${USE_SYSTEMD}" == X'YES' ]; then
+    echo "* Remove existing systemd service files."
+    rm -f ${SYSTEMD_SERVICE_DIR}/iredadmin.service &>/dev/null
+    rm -f ${SYSTEMD_SERVICE_DIR2}/iredadmin.service &>/dev/null
+    rm -f ${SYSTEMD_SERVICE_USER_DIR}/iredadmin.service &>/dev/null
+
+    echo "* Copy systemd service file: ${SYSTEMD_SERVICE_DIR}/iredadmin.service."
+    if [ X"${DISTRO}" == X'RHEL' ]; then
+        cp -f ${NEW_IRA_ROOT_DIR}/rc_scripts/systemd/rhel${DISTRO_VERSION}.service ${SYSTEMD_SERVICE_DIR}/iredadmin.service
+        perl -pi -e 's#/opt/www#$ENV{HTTPD_SERVERROOT}#g' ${SYSTEMD_SERVICE_DIR}/iredadmin.service
+        perl -pi -e 's#/usr/local/bin/uwsgi#$ENV{CMD_UWSGI}#g' ${SYSTEMD_SERVICE_DIR}/iredadmin.service
+
+        if [[ X"${UWSGI_HAS_PLUGINS}" == X"NO" ]]; then
+            _ini_file="${NEW_IRA_ROOT_DIR}/rc_scripts/uwsgi/rhel${DISTRO_VERSION}.ini"
+            if [[ -f ${_ini_file} ]]; then
+                perl -pi -e 's#^(plugins.*)##g' ${_ini_file}
+            fi
+        fi
+    elif [ X"${DISTRO}" == X'DEBIAN' -o X"${DISTRO}" == X'UBUNTU' ]; then
+        cp -f ${NEW_IRA_ROOT_DIR}/rc_scripts/systemd/debian.service ${SYSTEMD_SERVICE_DIR}/iredadmin.service
+        perl -pi -e 's#/opt/www#$ENV{HTTPD_SERVERROOT}#g' ${SYSTEMD_SERVICE_DIR}/iredadmin.service
+    fi
+
+    chmod -R 0644 ${SYSTEMD_SERVICE_DIR}/iredadmin.service
+    systemctl daemon-reload &>/dev/null
+else
+    if [ X"${DISTRO}" == X"FREEBSD" ]; then
+        cp ${NEW_IRA_ROOT_DIR}/rc_scripts/iredadmin.freebsd /usr/local/etc/rc.d/iredadmin
+        perl -pi -e 's#/opt/www#$ENV{HTTPD_SERVERROOT}#g' /usr/local/etc/rc.d/iredadmin
+
+        # Remove 'uwsgi_iredadmin_flags=' in /etc/rc.conf.local
+        if [ -f /etc/rc.conf.local ]; then
+            perl -pi -e 's#^uwsgi_iredadminflags=.*##g' /etc/rc.conf.local
+        fi
+    elif [ X"${DISTRO}" == X'OPENBSD' ]; then
+        cp ${NEW_IRA_ROOT_DIR}/rc_scripts/iredadmin.openbsd ${DIR_RC_SCRIPTS}/iredadmin
+        perl -pi -e 's#/opt/www#$ENV{HTTPD_SERVERROOT}#g' /etc/rc.d/iredadmin
+
+        cp -f ${NEW_IRA_ROOT_DIR}/rc_scripts/iredadmin.openbsd /etc/rc.d/iredadmin
+        chmod 0755 /etc/rc.d/iredadmin
+
+        # Remove 'uwsgi_flags=' in /etc/rc.conf.local
+        if [ -f /etc/rc.conf.local ]; then
+            perl -pi -e 's#^uwsgi_flags=.*iredadmin.*##g' /etc/rc.conf.local
+        fi
+    fi
+fi
+
+# Set owner and permission.
+chown -R ${IRA_HTTPD_USER}:${IRA_HTTPD_GROUP} ${NEW_IRA_ROOT_DIR}
+chmod -R 0555 ${NEW_IRA_ROOT_DIR}
+chmod 0400 ${NEW_IRA_ROOT_DIR}/settings.py
+
+echo "* Removing old symbol link ${IRA_ROOT_DIR}"
+rm -f ${IRA_ROOT_DIR}
+
+echo "* Creating symbol link ${IRA_ROOT_DIR} to ${NEW_IRA_ROOT_DIR}"
+cd ${HTTPD_SERVERROOT}
+ln -s ${name_new_version} iredadmin
+
+# Add missing setting parameters.
+if grep 'amavisd_enable_logging.*True.*' ${IRA_CONF_PY} &>/dev/null; then
+    add_missing_parameter 'amavisd_enable_policy_lookup' True 'Enable per-recipient spam policy, white/blacklist.'
+else
+    add_missing_parameter 'amavisd_enable_policy_lookup' False 'Enable per-recipient spam policy, white/blacklist.'
+fi
+
+if ! grep '^iredapd_' ${IRA_CONF_PY} &>/dev/null; then
+    add_missing_parameter 'iredapd_enabled' True 'Enable iRedAPD integration.'
+
+    # Get iredapd db password from /opt/iredapd/settings.py.
+    if [ -f /opt/iredapd/settings.py ]; then
+        grep '^iredapd_db_' /opt/iredapd/settings.py >> ${IRA_CONF_PY}
+        perl -pi -e 's#iredapd_db_server#iredapd_db_host#g' ${IRA_CONF_PY}
+    else
+        # Check backend.
+        if egrep '^backend.*pgsql' ${IRA_CONF_PY} &>/dev/null; then
+            export IREDAPD_DB_PORT='5432'
+        else
+            export IREDAPD_DB_PORT='3306'
+        fi
+
+        add_missing_parameter 'iredapd_db_host' '127.0.0.1'
+        add_missing_parameter 'iredapd_db_port' ${IREDAPD_DB_PORT}
+        add_missing_parameter 'iredapd_db_name' 'iredapd'
+        add_missing_parameter 'iredapd_db_user' 'iredapd'
+        add_missing_parameter 'iredapd_db_password' 'password'
+    fi
+fi
+perl -pi -e 's#iredapd_db_server#iredapd_db_host#g' ${IRA_CONF_PY}
+
+if ! grep '^fail2ban_' ${IRA_CONF_PY} &>/dev/null; then
+    # Try to get password of SQL user `fail2ban`.
+    if egrep '^backend.*(mysql|ldap)' ${IRA_CONF_PY} &>/dev/null; then
+        _my_cnf='/root/.my.cnf-fail2ban'
+        if [ -f ${_my_cnf} ]; then
+            _host="$(grep '^host=' ${_my_cnf} | awk -F'host=' '{print $2}' | strip_quotes)"
+            _port="$(grep '^port=' ${_my_cnf} | awk -F'port=' '{print $2}' | strip_quotes)"
+            _user="$(grep '^user=' ${_my_cnf} | awk -F'user=' '{print $2}' | strip_quotes)"
+            _password="$(grep '^password=' ${_my_cnf} | awk -F'password=' '{print $2}' | strip_quotes)"
+
+            [ X"${_host}" == X'' ] && _host='127.0.0.1'
+            [ X"${_port}" == X'' ] && _port='3306'
+        fi
+    elif egrep '^backend.*pgsql' ${IRA_CONF_PY} &>/dev/null; then
+        # Absolute path to ~/.pgpass
+        #   - RHEL:             /var/lib/pgsql/.pgpass
+        #   - Debian/Ubuntu:    /var/lib/postgresql/.pgpass
+        #   - FreeBSD:          /var/db/postgres/.pgpass
+        #   - OpenBSD:          /var/postgresql/.pgpass
+        for dir in \
+            /var/lib/pgsql \
+            /var/lib/postgresql \
+            /var/db/postgres \
+            /var/postgresql; do
+            _pgpass="${dir}/.pgpass"
+            if [ -f ${_pgpass} ]; then
+                if grep ':fail2ban:' ${_pgpass} &>/dev/null; then
+                    _host="127.0.0.1"
+                    _port="5432"
+                    _user="fail2ban"
+                    _password="$(grep ':fail2ban:' ${_pgpass} | awk -F':' '{print $NF}')"
+                    break
+                fi
+            fi
+        done
+    fi
+
+    if [ X"${_host}" != X'' ] && \
+        [ X"${_port}" != X'' ] && \
+        [ X"${_user}" != X'' ] && \
+        [ X"${_password}" != X'' ]; then
+        echo "* Enable Fail2ban SQL integration."
+        add_missing_parameter 'fail2ban_enabled' 'True'
+        add_missing_parameter 'fail2ban_db_host' "${_host}"
+        add_missing_parameter 'fail2ban_db_port' "${_port}"
+        add_missing_parameter 'fail2ban_db_name' "fail2ban"
+        add_missing_parameter 'fail2ban_db_user' "${_user}"
+        add_missing_parameter 'fail2ban_db_password' "${_password}"
+    fi
+fi
+
+# FreeBSD uses /var/run/log for syslog.
+if [ X"${DISTRO}" == X'FREEBSD' ]; then
+    add_missing_parameter 'SYSLOG_SERVER' '/var/run/log'
+fi
+
+#
+# Enable mlmmj integration
+#
+if [ -e /opt/mlmmjadmin ]; then
+    echo "* Enable mlmmj integration."
+    # Force to use backend `bk_none`.
+    perl -pi -e 's#^(backend_api).*#${1} = "bk_none"#g' /opt/mlmmjadmin/settings.py
+
+    if egrep '^backend.*(ldap)' ${IRA_CONF_PY} &>/dev/null; then
+        perl -pi -e 's#^(backend_cli).*#${1} = "bk_iredmail_ldap"#g' /opt/mlmmjadmin/settings.py
+    else
+        perl -pi -e 's#^(backend_cli).*#${1} = "bk_iredmail_sql"#g' /opt/mlmmjadmin/settings.py
+    fi
+
+    # Add parameter `mlmmjadmin_api_auth_token` if missing
+    if ! grep '^mlmmjadmin_api_auth_token' ${IRA_CONF_PY} >/dev/null; then
+        # Get first api auth token
+        token=$(grep '^api_auth_tokens' /opt/mlmmjadmin/settings.py | awk -F"[=\']" '{print $3}' | tr -d '\n')
+        echo -e "\nmlmmjadmin_api_auth_token = '${token}'" >> ${IRA_CONF_PY}
+    fi
+
+    echo "* Restarting service: mlmmjadmin."
+    restart_service mlmmjadmin
+fi
+
+# Change old parameter names to the new ones:
+#
+#   - ADDITION_USER_SERVICES -> ADDITIONAL_ENABLED_USER_SERVICES
+#   - LDAP_SERVER_NAME -> LDAP_SERVER_PRODUCT_NAME
+perl -pi -e 's#ADDITION_USER_SERVICES#ADDITIONAL_ENABLED_USER_SERVICES#g' ${IRA_CONF_PY}
+perl -pi -e 's#LDAP_SERVER_NAME#LDAP_SERVER_PRODUCT_NAME#g' ${IRA_CONF_PY}
+
+# Remove deprecated setting: ENABLE_SELF_SERVICE, it's now a per-domain setting.
+perl -pi -e 's#^(ENABLE_SELF_SERVICE.*)##g' ${IRA_CONF_PY}
+
+
+# Dependent packages.
+export REQUIRED_PKGS=""
+export PIP3_MODS=""
+# Python modules.
+export PKG_PY_PIP='python3-pip'
+export PKG_PY_LDAP='python3-ldap'
+export PKG_PY_MYSQL='python3-pymysql'
+export PKG_PY_PGSQL='python3-psycopg2'
+export PKG_PY_JSON='python3-simplejson'
+export PKG_PY_DNS='python3-dnspython'
+export PKG_PY_REQUESTS='python3-requests'
+export PKG_PY_JINJA='python3-jinja2'
+# Python modules installed with pip3: uwsgi.
+
+if [ X"${DISTRO}" == X'RHEL' ]; then
+    if [ X"${DISTRO_VERSION}" == X'7' ]; then
+        export PKG_PY_MYSQL='python36-PyMySQL'
+        export PKG_PY_JSON='python36-simplejson'
+        export PKG_PY_JINJA='python36-jinja2'
+        export REQUIRED_PKGS="${REQUIRED_PKGS} uwsgi uwsgi-plugin-python36 uwsgi-plugin-syslog"
+
+        if rpm -q mod_wsgi &>/dev/null; then
+            remove_pkg mod_wsgi
+            export REQUIRED_PKGS="${REQUIRED_PKGS} python3-mod_wsgi"
+        fi
+
+    else
+        if [ ! -x ${CMD_UWSGI} ]; then
+            # gcc is required to install uwsgi.
+            export REQUIRED_PKGS="${REQUIRED_PKGS} python3-devel python3-pip gcc"
+            export PIP3_MODS="${PIP3_MODS} uwsgi"
+        fi
+    fi
+
+    export PKG_PY_DNS='python3-dns'
+elif [ X"${DISTRO}" == X'DEBIAN' -o X"${DISTRO}" == X'UBUNTU' ]; then
+    export REQUIRED_PKGS="${REQUIRED_PKGS} uwsgi-core uwsgi-plugin-python3"
+
+    if [ X"${DISTRO_VERSION}" == X'9' ]; then
+        export PKG_PY_LDAP='python3-pyldap'
+    else
+        export PKG_PY_LDAP='python3-ldap'
+    fi
+elif [ X"${DISTRO}" == X'OPENBSD' ]; then
+    export PKG_PY_PIP='py3-pip'
+    export PKG_PY_JSON='py3-simplejson'
+    export PKG_PY_DNS='py3-dnspython'
+    export PKG_PY_REQUESTS='py3-requests'
+    export PKG_PY_JINJA='py3-jinja2'
+    if [ X"${DISTRO_VERSION}" == X'6.6' -o X"${DISTRO_VERSION}" == X'6.7' ]; then
+        export PKG_PY_MYSQL='py3-mysqlclient'
+    else
+        export PKG_PY_MYSQL='py3-pymysql'
+    fi
+
+    if [ ! -x ${CMD_UWSGI} ]; then
+        export PIP3_MODS="${PIP3_MODS} uwsgi"
+    fi
+elif [ X"${DISTRO}" == X'FREEBSD' ]; then
+    export PKG_PY_PIP='devel/py-pip'
+    export PKG_UWSGI="www/uwsgi"
+    export PKG_PY_JSON='devel/py-simplejson'
+    export PKG_PY_DNS='dns/py-dnspython'
+    export PKG_PY_REQUESTS='www/py-requests'
+    export PKG_PY_JINJA='devel/py-Jinja2'
+
+    if [ ! -x ${CMD_UWSGI} ]; then
+        export REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_UWSGI}"
+    fi
+fi
+
+echo "* Check and install required packages."
+if egrep '^backend.*ldap' ${IRA_CONF_PY} &>/dev/null; then
+    [ X"$(has_python_module ldap)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_LDAP}"
+    [ X"$(has_python_module pymysql)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_MYSQL}"
+elif egrep '^backend.*mysql' ${IRA_CONF_PY} &>/dev/null; then
+    [ X"$(has_python_module pymysql)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_MYSQL}"
+elif egrep '^backend.*pgsql' ${IRA_CONF_PY} &>/dev/null; then
+    [ X"$(has_python_module psycopg2)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_PGSQL}"
+fi
+[ X"$(has_python_module pip)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_PIP}"
+[ X"$(has_python_module simplejson)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_JSON}"
+[ X"$(has_python_module dns)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_DNS}"
+[ X"$(has_python_module requests)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_REQUESTS}"
+if [ X"$(has_python_module web)" == X'NO' ]; then
+    PIP3_MODS="${PIP3_MODS} web.py>=0.61"
+else # Verify module version.
+    _webpy_ver=$(${CMD_PYTHON3} -c "import web; print(web.__version__)")
+    if echo ${_webpy_ver} | grep '^0\.[45]' &>/dev/null; then
+        PIP3_MODS="${PIP3_MODS} web.py>=0.61"
+    fi
+fi
+[ X"$(has_python_module jinja2)" == X'NO' ] && REQUIRED_PKGS="${REQUIRED_PKGS} ${PKG_PY_JINJA}"
+
+if [ X"${REQUIRED_PKGS}" != X'' ]; then
+    install_pkg ${REQUIRED_PKGS}
+    if [ X"$?" != X'0' ]; then
+        echo "Package installation failed, please check console output and fix it manually."
+        exist 255
+    fi
+fi
+
+if [ X"${PIP3_MODS}" != X'' ]; then
+    ${CMD_PIP3} install -U ${PIP3_MODS}
+    if [ X"$?" != X'0' ]; then
+        echo "Package installation failed, please check console output and fix it manually."
+        exist 255
+    fi
+fi
+
+#------------------------------------------
+# Add new SQL tables, drop deprecated ones.
+#
+export ira_db_host="$(get_iredadmin_setting 'iredadmin_db_host')"
+export ira_db_port="$(get_iredadmin_setting 'iredadmin_db_port')"
+export ira_db_name="$(get_iredadmin_setting 'iredadmin_db_name')"
+export ira_db_user="$(get_iredadmin_setting 'iredadmin_db_user')"
+export ira_db_password="$(get_iredadmin_setting 'iredadmin_db_password')"
+
+#
+# Update sql tables
+#
+psql_conn="psql -h ${ira_db_host} \
+                -p ${ira_db_port} \
+                -U ${ira_db_user} \
+                -d ${ira_db_name}"
+
+if egrep '^backend.*(mysql|ldap)' ${IRA_CONF_PY} &>/dev/null; then
+    echo "* Check SQL tables, and add missed ones - if there's any"
+    ${CMD_MYSQL} ${ira_db_name} -e "SOURCE ${IRA_ROOT_DIR}/SQL/iredadmin.mysql"
+    ${CMD_MYSQL} ${ira_db_name} -e "ALTER TABLE log MODIFY COLUMN msg TEXT;"
+
+    # Add column `tracking.id`.
+    ${CMD_MYSQL} ${ira_db_name} -e "DESC tracking \G" | grep 'Field: id' &>/dev/null
+    if [ X"$?" != X'0' ]; then
+        ${CMD_MYSQL} ${ira_db_name} -e "ALTER TABLE tracking ADD COLUMN id BIGINT(20) UNSIGNED AUTO_INCREMENT PRIMARY KEY;"
+    fi
+
+    # Set column `id` to `PRIMARY KEY`
+    _tables='deleted_mailboxes updatelog log tracking'
+    for _table in ${_tables}; do
+        ${CMD_MYSQL} ${ira_db_name} -e "DESC ${_table}" | grep '^id.*PRI.*auto_increment' &>/dev/null
+
+        if [ X"$?" != X'0' ]; then
+            ${CMD_MYSQL} ${ira_db_name} -e "ALTER TABLE ${_table} ADD PRIMARY KEY (id)"
+        fi
+    done
+
+elif egrep '^backend.*pgsql' ${IRA_CONF_PY} &>/dev/null; then
+    export PGPASSWORD="${ira_db_password}"
+
+    # Allow log.msg to store long text.
+    ${psql_conn} <<EOF
+ALTER TABLE log ALTER COLUMN msg TYPE TEXT;
+EOF
+
+    # SQL table: tracking.
+    ${psql_conn} -c '\d' | grep '\<tracking\>' &>/dev/null
+    if [ X"$?" != X'0' ]; then
+        echo "* [SQL] Add new table: iredadmin.tracking."
+
+        ${psql_conn} <<EOF
+CREATE TABLE tracking (
+    id SERIAL PRIMARY KEY,
+    k VARCHAR(50) NOT NULL,
+    v TEXT,
+    time TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE UNIQUE INDEX idx_tracking_k ON tracking (k);
+EOF
+    fi
+
+    # Set column `tracking.id` to `PRIMARY KEY`
+
+    # SQL table: domain_ownership.
+    ${psql_conn} -c '\d' | grep '\<domain_ownership\>' &>/dev/null
+    if [ X"$?" != X'0' ]; then
+        echo "* [SQL] Add new table: iredadmin.domain_ownership."
+
+        ${psql_conn} <<EOF
+CREATE TABLE domain_ownership (
+    id SERIAL PRIMARY KEY,
+    admin VARCHAR(255) NOT NULL DEFAULT '',
+    domain VARCHAR(255) NOT NULL DEFAULT '',
+    alias_domain VARCHAR(255) NOT NULL DEFAULT '',
+    verify_code VARCHAR(100) NOT NULL DEFAULT '',
+    verified INT2 NOT NULL DEFAULT 0,
+    message TEXT,
+    last_verify TIMESTAMP NULL DEFAULT NULL,
+    expire INT DEFAULT 0
+);
+CREATE UNIQUE INDEX idx_ownership_1 ON domain_ownership (admin, domain, alias_domain);
+CREATE INDEX idx_ownership_2 ON domain_ownership (verified);
+EOF
+    fi
+
+    # SQL table: newsletter_subunsub_confirms.
+    ${psql_conn} -c '\d' | grep '\<newsletter_subunsub_confirms\>' &>/dev/null
+    if [ X"$?" != X'0' ]; then
+        echo "* [SQL] Add new table: iredadmin.newsletter_subunsub_confirms."
+
+        _sql="$(cat ${IRA_ROOT_DIR}/SQL/snippets/newsletter_subunsub_confirms.pgsql)"
+        ${psql_conn} <<EOF
+${_sql}
+EOF
+        unset _sql
+    fi
+
+    # SQL table: settings.
+    ${psql_conn} -c '\d' | grep '\<settings\>' &>/dev/null
+    if [ X"$?" != X'0' ]; then
+        echo "* [SQL] Add new table: iredadmin.settings."
+
+        _sql="$(cat ${IRA_ROOT_DIR}/SQL/snippets/settings.pgsql)"
+        ${psql_conn} <<EOF
+${_sql}
+EOF
+        unset _sql
+    fi
+fi
+
+#------------------------------
+# Cron job.
+#
+[[ -d ${CRON_SPOOL_DIR} ]] || mkdir -p ${CRON_SPOOL_DIR} &>/dev/null
+if [[ ! -f ${CRON_FILE_ROOT} ]]; then
+    touch ${CRON_FILE_ROOT} &>/dev/null
+    chmod 0600 ${CRON_FILE_ROOT} &>/dev/null
+fi
+
+# cron job: clean up database.
+if ! grep 'iredadmin/tools/cleanup_db.py' ${CRON_FILE_ROOT} &>/dev/null; then
+    cat >> ${CRON_FILE_ROOT} <<EOF
+# iRedAdmin: Clean up sql database.
+1   *   *   *   *   ${CMD_PYTHON3} ${IRA_ROOT_DIR}/tools/cleanup_db.py &>/dev/null
+EOF
+fi
+
+# cron job: clean up database.
+if ! grep 'iredadmin/tools/delete_mailboxes.py' ${CRON_FILE_ROOT} &>/dev/null; then
+    cat >> ${CRON_FILE_ROOT} <<EOF
+# iRedAdmin: Remove mailboxes which are scheduled to be removed.
+1   3   *   *   *   ${CMD_PYTHON3} ${IRA_ROOT_DIR}/tools/delete_mailboxes.py
+EOF
+fi
+
+echo "* Replace py2 by py3 in cron jobs."
+perl -pi -e 's#(.*) python (.*/iredadmin/tools/.*)#${1} $ENV{CMD_PYTHON3} ${2}#' ${CRON_FILE_ROOT}
+perl -pi -e 's#(.*) python2 (.*/iredadmin/tools/.*)#${1} $ENV{CMD_PYTHON3} ${2}#' ${CRON_FILE_ROOT}
+perl -pi -e 's#(.*)/usr/bin/python (.*/iredadmin/tools/.*)#${1}$ENV{CMD_PYTHON3} ${2}#' ${CRON_FILE_ROOT}
+perl -pi -e 's#(.*)/usr/bin/python2 (.*/iredadmin/tools/.*)#${1}$ENV{CMD_PYTHON3} ${2}#' ${CRON_FILE_ROOT}
+perl -pi -e 's#(.*)/usr/local/bin/python (.*/iredadmin/tools/.*)#${1}$ENV{CMD_PYTHON3} ${2}#' ${CRON_FILE_ROOT}
+perl -pi -e 's#(.*)/usr/local/bin/python2 (.*/iredadmin/tools/.*)#${1}$ENV{CMD_PYTHON3} ${2}#' ${CRON_FILE_ROOT}
+
+echo "* Clean up."
+cd ${NEW_IRA_ROOT_DIR}/
+rm -f settings.pyc settings.pyo tools/settings.py
+if [[ -f ${NEW_IRA_ROOT_DIR}/libs/form_utils.py ]]; then
+    # Not a trial license.
+    cd ${NEW_IRA_ROOT_DIR}
+    find . -name '*.so' | xargs rm -f {}
+    cd - &>/dev/null
+fi
+
+# Delete all sessions to force admins to re-login.
+cd ${NEW_IRA_ROOT_DIR}/tools/
+${CMD_PYTHON3} delete_sessions.py
+
+echo "* iRedAdmin has been successfully upgraded."
+restart_web_service
+
+# Enable and restart service
+enable_service iredadmin
+restart_service iredadmin
+
+echo "* Upgrading completed."
+
+cat <<EOF
+<<< NOTE >>> If iRedAdmin doesn't work as expected, please post your issue in
+<<< NOTE >>> our online support forum: http://www.iredmail.org/forum/
+EOF

web/__init__.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""web.py: makes web apps (http://webpy.org)"""
+
+from . import (  # noqa: F401
+    db,
+    debugerror,
+    form,
+    http,
+    httpserver,
+    net,
+    session,
+    template,
+    utils,
+    webapi,
+    wsgi,
+)
+from .application import *  # noqa: F401,F403
+from .db import *  # noqa: F401,F403
+from .debugerror import *  # noqa: F401,F403
+from .http import *  # noqa: F401,F403
+from .httpserver import *  # noqa: F401,F403
+from .net import *  # noqa: F401,F403
+from .utils import *  # noqa: F401,F403
+from .webapi import *  # noqa: F401,F403
+from .wsgi import *  # noqa: F401,F403
+
+__version__ = "0.62"
+__author__ = [
+    "Aaron Swartz <me@aaronsw.com>",
+    "Anand Chitipothu <anandology@gmail.com>",
+]
+__license__ = "public domain"
+__contributors__ = "see http://webpy.org/changes"

web/application.py

@@ -0,0 +1,813 @@
+"""
+Web application
+(from web.py)
+"""
+
+import itertools
+import os
+import sys
+import traceback
+import wsgiref.handlers
+from importlib import reload
+from inspect import isclass
+from io import BytesIO
+from urllib.parse import unquote, urlencode, urlparse
+
+from . import browser, httpserver, utils
+from . import webapi as web
+from . import wsgi
+from .debugerror import debugerror
+from .py3helpers import iteritems
+from .utils import lstrips
+
+__all__ = [
+    "application",
+    "auto_application",
+    "subdir_application",
+    "subdomain_application",
+    "loadhook",
+    "unloadhook",
+    "autodelegate",
+]
+
+
+class application:
+    """
+    Application to delegate requests based on path.
+
+        >>> urls = ("/hello", "hello")
+        >>> app = application(urls, globals())
+        >>> class hello:
+        ...     def GET(self): return "hello"
+        >>>
+        >>> app.request("/hello").data
+        'hello'
+    """
+
+    # PY3DOCTEST: b'hello'
+
+    def __init__(self, mapping=(), fvars={}, autoreload=None):
+        if autoreload is None:
+            autoreload = web.config.get("debug", False)
+        self.init_mapping(mapping)
+        self.fvars = fvars
+        self.processors = []
+
+        self.add_processor(loadhook(self._load))
+        self.add_processor(unloadhook(self._unload))
+
+        if autoreload:
+
+            def main_module_name():
+                mod = sys.modules["__main__"]
+                file = getattr(
+                    mod, "__file__", None
+                )  # make sure this works even from python interpreter
+                return file and os.path.splitext(os.path.basename(file))[0]
+
+            def modname(fvars):
+                """find name of the module name from fvars."""
+                file, name = fvars.get("__file__"), fvars.get("__name__")
+                if file is None or name is None:
+                    return None
+
+                if name == "__main__":
+                    # Since the __main__ module can't be reloaded, the module has
+                    # to be imported using its file name.
+                    name = main_module_name()
+                return name
+
+            mapping_name = utils.dictfind(fvars, mapping)
+            module_name = modname(fvars)
+
+            def reload_mapping():
+                """loadhook to reload mapping and fvars."""
+                mod = __import__(module_name, None, None, [""])
+                mapping = getattr(mod, mapping_name, None)
+                if mapping:
+                    self.fvars = mod.__dict__
+                    self.init_mapping(mapping)
+
+            self.add_processor(loadhook(Reloader()))
+            if mapping_name and module_name:
+                # when app is ran as part of a package, this puts the app into
+                # `sys.modules` correctly, otherwise the first change to the
+                # app module will not be picked up by Reloader
+                reload_mapping()
+
+                self.add_processor(loadhook(reload_mapping))
+
+            # load __main__ module usings its filename, so that it can be reloaded.
+            if main_module_name() and "__main__" in sys.argv:
+                try:
+                    __import__(main_module_name())
+                except ImportError:
+                    pass
+
+    def _load(self):
+        web.ctx.app_stack.append(self)
+
+    def _unload(self):
+        web.ctx.app_stack = web.ctx.app_stack[:-1]
+
+        if web.ctx.app_stack:
+            # this is a sub-application, revert ctx to earlier state.
+            oldctx = web.ctx.get("_oldctx")
+            if oldctx:
+                web.ctx.home = oldctx.home
+                web.ctx.homepath = oldctx.homepath
+                web.ctx.path = oldctx.path
+                web.ctx.fullpath = oldctx.fullpath
+
+    def _cleanup(self):
+        # Threads can be recycled by WSGI servers.
+        # Clearing up all thread-local state to avoid interefereing with subsequent requests.
+        utils.ThreadedDict.clear_all()
+
+    def init_mapping(self, mapping):
+        self.mapping = list(utils.group(mapping, 2))
+
+    def add_mapping(self, pattern, classname):
+        self.mapping.append((pattern, classname))
+
+    def add_processor(self, processor):
+        """
+        Adds a processor to the application.
+
+            >>> urls = ("/(.*)", "echo")
+            >>> app = application(urls, globals())
+            >>> class echo:
+            ...     def GET(self, name): return name
+            ...
+            >>>
+            >>> def hello(handler): return "hello, " +  handler()
+            ...
+            >>> app.add_processor(hello)
+            >>> app.request("/web.py").data
+            'hello, web.py'
+        """
+        # PY3DOCTEST: b'hello, web.py'
+        self.processors.append(processor)
+
+    def request(
+        self,
+        localpart="/",
+        method="GET",
+        data=None,
+        host="0.0.0.0:8080",
+        headers=None,
+        https=False,
+        **kw,
+    ):
+        """Makes request to this application for the specified path and method.
+        Response will be a storage object with data, status and headers.
+
+            >>> urls = ("/hello", "hello")
+            >>> app = application(urls, globals())
+            >>> class hello:
+            ...     def GET(self):
+            ...         web.header('Content-Type', 'text/plain')
+            ...         return "hello"
+            ...
+            >>> response = app.request("/hello")
+            >>> response.data
+            'hello'
+            >>> response.status
+            '200 OK'
+            >>> response.headers['Content-Type']
+            'text/plain'
+
+        To use https, use https=True.
+
+            >>> urls = ("/redirect", "redirect")
+            >>> app = application(urls, globals())
+            >>> class redirect:
+            ...     def GET(self): raise web.seeother("/foo")
+            ...
+            >>> response = app.request("/redirect")
+            >>> response.headers['Location']
+            'http://0.0.0.0:8080/foo'
+            >>> response = app.request("/redirect", https=True)
+            >>> response.headers['Location']
+            'https://0.0.0.0:8080/foo'
+
+        The headers argument specifies HTTP headers as a mapping object
+        such as a dict.
+
+            >>> urls = ('/ua', 'uaprinter')
+            >>> class uaprinter:
+            ...     def GET(self):
+            ...         return 'your user-agent is ' + web.ctx.env['HTTP_USER_AGENT']
+            ...
+            >>> app = application(urls, globals())
+            >>> app.request('/ua', headers = {
+            ...      'User-Agent': 'a small jumping bean/1.0 (compatible)'
+            ... }).data
+            'your user-agent is a small jumping bean/1.0 (compatible)'
+
+        """
+        # PY3DOCTEST: b'hello'
+        # PY3DOCTEST: b'your user-agent is a small jumping bean/1.0 (compatible)'
+        _p = urlparse(localpart)
+        path = _p.path
+        maybe_query = _p.query
+
+        query = maybe_query or ""
+
+        if "env" in kw:
+            env = kw["env"]
+        else:
+            env = {}
+        env = dict(
+            env,
+            HTTP_HOST=host,
+            REQUEST_METHOD=method,
+            PATH_INFO=path,
+            QUERY_STRING=query,
+            HTTPS=str(https),
+        )
+        headers = headers or {}
+
+        for k, v in headers.items():
+            env["HTTP_" + k.upper().replace("-", "_")] = v
+
+        if "HTTP_CONTENT_LENGTH" in env:
+            env["CONTENT_LENGTH"] = env.pop("HTTP_CONTENT_LENGTH")
+
+        if "HTTP_CONTENT_TYPE" in env:
+            env["CONTENT_TYPE"] = env.pop("HTTP_CONTENT_TYPE")
+
+        if method not in ["HEAD", "GET"]:
+            data = data or ""
+
+            if isinstance(data, dict):
+                q = urlencode(data)
+            else:
+                q = data
+
+            env["wsgi.input"] = BytesIO(q.encode("utf-8"))
+            # if not env.get('CONTENT_TYPE', '').lower().startswith('multipart/') and 'CONTENT_LENGTH' not in env:
+            if "CONTENT_LENGTH" not in env:
+                env["CONTENT_LENGTH"] = len(q)
+        response = web.storage()
+
+        def start_response(status, headers):
+            response.status = status
+            response.headers = dict(headers)
+            response.header_items = headers
+
+        data = self.wsgifunc()(env, start_response)
+        response.data = b"".join(data)
+        return response
+
+    def browser(self):
+        return browser.AppBrowser(self)
+
+    def handle(self):
+        fn, args = self._match(self.mapping, web.ctx.path)
+        return self._delegate(fn, self.fvars, args)
+
+    def handle_with_processors(self):
+        def process(processors):
+            try:
+                if processors:
+                    p, processors = processors[0], processors[1:]
+                    return p(lambda: process(processors))
+                else:
+                    return self.handle()
+            except web.HTTPError:
+                raise
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except:
+                print(traceback.format_exc(), file=web.debug)
+                raise self.internalerror()
+
+        # processors must be applied in the reverse order. (??)
+        return process(self.processors)
+
+    def wsgifunc(self, *middleware):
+        """Returns a WSGI-compatible function for this application."""
+
+        def peep(iterator):
+            """Peeps into an iterator by doing an iteration
+            and returns an equivalent iterator.
+            """
+            # wsgi requires the headers first
+            # so we need to do an iteration
+            # and save the result for later
+            try:
+                firstchunk = next(iterator)
+            except StopIteration:
+                firstchunk = ""
+
+            return itertools.chain([firstchunk], iterator)
+
+        def wsgi(env, start_resp):
+            # clear threadlocal to avoid interference of previous requests
+            self._cleanup()
+
+            self.load(env)
+            try:
+                # allow uppercase methods only
+                if web.ctx.method.upper() != web.ctx.method:
+                    raise web.nomethod()
+
+                result = self.handle_with_processors()
+                if result and hasattr(result, "__next__"):
+                    result = peep(result)
+                else:
+                    result = [result]
+            except web.HTTPError as e:
+                result = [e.data]
+
+            def build_result(result):
+                for r in result:
+                    if isinstance(r, bytes):
+                        yield r
+                    else:
+                        yield str(r).encode("utf-8")
+
+            result = build_result(result)
+
+            status, headers = web.ctx.status, web.ctx.headers
+            start_resp(status, headers)
+
+            def cleanup():
+                self._cleanup()
+                yield b""  # force this function to be a generator
+
+            return itertools.chain(result, cleanup())
+
+        for m in middleware:
+            wsgi = m(wsgi)
+
+        return wsgi
+
+    def run(self, *middleware):
+        """
+        Starts handling requests. If called in a CGI or FastCGI context, it will follow
+        that protocol. If called from the command line, it will start an HTTP
+        server on the port named in the first command line argument, or, if there
+        is no argument, on port 8080.
+
+        `middleware` is a list of WSGI middleware which is applied to the resulting WSGI
+        function.
+        """
+        return wsgi.runwsgi(self.wsgifunc(*middleware))
+
+    def stop(self):
+        """Stops the http server started by run."""
+        if httpserver.server:
+            httpserver.server.stop()
+            httpserver.server = None
+
+    def cgirun(self, *middleware):
+        """
+        Return a CGI handler. This is mostly useful with Google App Engine.
+        There you can just do:
+
+            main = app.cgirun()
+        """
+        wsgiapp = self.wsgifunc(*middleware)
+
+        try:
+            from google.appengine.ext.webapp.util import run_wsgi_app
+
+            return run_wsgi_app(wsgiapp)
+        except ImportError:
+            # we're not running from within Google App Engine
+            return wsgiref.handlers.CGIHandler().run(wsgiapp)
+
+    def gaerun(self, *middleware):
+        """
+        Starts the program in a way that will work with Google app engine,
+        no matter which version you are using (2.5 / 2.7)
+
+        If it is 2.5, just normally start it with app.gaerun()
+
+        If it is 2.7, make sure to change the app.yaml handler to point to the
+        global variable that contains the result of app.gaerun()
+
+        For example:
+
+        in app.yaml (where code.py is where the main code is located)
+
+            handlers:
+            - url: /.*
+              script: code.app
+
+        Make sure that the app variable is globally accessible
+        """
+        wsgiapp = self.wsgifunc(*middleware)
+        try:
+            # check what version of python is running
+            version = sys.version_info[:2]
+            major = version[0]
+            minor = version[1]
+
+            if major != 2:
+                raise OSError("Google App Engine only supports python 2.5 and 2.7")
+
+            # if 2.7, return a function that can be run by gae
+            if minor == 7:
+                return wsgiapp
+            # if 2.5, use run_wsgi_app
+            elif minor == 5:
+                from google.appengine.ext.webapp.util import run_wsgi_app
+
+                return run_wsgi_app(wsgiapp)
+            else:
+                raise OSError("Not a supported platform, use python 2.5 or 2.7")
+        except ImportError:
+            return wsgiref.handlers.CGIHandler().run(wsgiapp)
+
+    def load(self, env):
+        """Initializes ctx using env."""
+        ctx = web.ctx
+        ctx.clear()
+        ctx.status = "200 OK"
+        ctx.headers = []
+        ctx.output = ""
+        ctx.environ = ctx.env = env
+        ctx.host = env.get("HTTP_HOST")
+
+        if env.get("wsgi.url_scheme") in ["http", "https"]:
+            ctx.protocol = env["wsgi.url_scheme"]
+        elif env.get("HTTPS", "").lower() in ["on", "true", "1"]:
+            ctx.protocol = "https"
+        else:
+            ctx.protocol = "http"
+        ctx.homedomain = ctx.protocol + "://" + env.get("HTTP_HOST", "[unknown]")
+        ctx.homepath = os.environ.get("REAL_SCRIPT_NAME", env.get("SCRIPT_NAME", ""))
+        ctx.home = ctx.homedomain + ctx.homepath
+        # @@ home is changed when the request is handled to a sub-application.
+        # @@ but the real home is required for doing absolute redirects.
+        ctx.realhome = ctx.home
+        ctx.ip = env.get("REMOTE_ADDR")
+        ctx.method = env.get("REQUEST_METHOD")
+        try:
+            ctx.path = bytes(env.get("PATH_INFO"), "latin1").decode("utf8")
+        except UnicodeDecodeError:  # If there are Unicode characters...
+            ctx.path = env.get("PATH_INFO")
+
+        # http://trac.lighttpd.net/trac/ticket/406 requires:
+        if env.get("SERVER_SOFTWARE", "").startswith(("lighttpd/", "nginx/")):
+            ctx.path = lstrips(env.get("REQUEST_URI").split("?")[0], ctx.homepath)
+            # Apache and CherryPy webservers unquote urls but lighttpd and nginx do not.
+            # Unquote explicitly for lighttpd and nginx to make ctx.path uniform across
+            # all servers.
+            ctx.path = unquote(ctx.path)
+
+        if env.get("QUERY_STRING"):
+            ctx.query = "?" + env.get("QUERY_STRING", "")
+        else:
+            ctx.query = ""
+
+        ctx.fullpath = ctx.path + ctx.query
+
+        for k, v in iteritems(ctx):
+            # convert all string values to unicode values and replace
+            # malformed data with a suitable replacement marker.
+            if isinstance(v, bytes):
+                ctx[k] = v.decode("utf-8", "replace")
+
+        # status must always be str
+        ctx.status = "200 OK"
+
+        ctx.app_stack = []
+
+    def _delegate(self, f, fvars, args=[]):
+        def handle_class(cls):
+            meth = web.ctx.method
+            if meth == "HEAD" and not hasattr(cls, meth):
+                meth = "GET"
+            if not hasattr(cls, meth):
+                raise web.nomethod(cls)
+            tocall = getattr(cls(), meth)
+            return tocall(*args)
+
+        if f is None:
+            raise web.notfound()
+        elif isinstance(f, application):
+            return f.handle_with_processors()
+        elif isclass(f):
+            return handle_class(f)
+        elif isinstance(f, str):
+            if f.startswith("redirect "):
+                url = f.split(" ", 1)[1]
+                if web.ctx.method == "GET":
+                    x = web.ctx.env.get("QUERY_STRING", "")
+                    if x:
+                        url += "?" + x
+                raise web.redirect(url)
+            elif "." in f:
+                mod, cls = f.rsplit(".", 1)
+                mod = __import__(mod, None, None, [""])
+                cls = getattr(mod, cls)
+            else:
+                cls = fvars[f]
+            return handle_class(cls)
+        elif hasattr(f, "__call__"):
+            return f()
+        else:
+            return web.notfound()
+
+    def _match(self, mapping, value):
+        for pat, what in mapping:
+            if isinstance(what, application):
+                if value.startswith(pat):
+                    f = lambda: self._delegate_sub_application(pat, what)
+                    return f, None
+                else:
+                    continue
+            elif isinstance(what, str):
+                what, result = utils.re_subm(rf"^{pat}\Z", what, value)
+            else:
+                result = utils.re_compile(rf"^{pat}\Z").match(value)
+
+            if result:  # it's a match
+                return what, [x for x in result.groups()]
+        return None, None
+
+    def _delegate_sub_application(self, dir, app):
+        """Deletes request to sub application `app` rooted at the directory `dir`.
+        The home, homepath, path and fullpath values in web.ctx are updated to mimic request
+        to the subapp and are restored after it is handled.
+
+        @@Any issues with when used with yield?
+        """
+        web.ctx._oldctx = web.storage(web.ctx)
+        web.ctx.home += dir
+        web.ctx.homepath += dir
+        web.ctx.path = web.ctx.path[len(dir) :]
+        web.ctx.fullpath = web.ctx.fullpath[len(dir) :]
+        return app.handle_with_processors()
+
+    def get_parent_app(self):
+        if self in web.ctx.app_stack:
+            index = web.ctx.app_stack.index(self)
+            if index > 0:
+                return web.ctx.app_stack[index - 1]
+
+    def notfound(self):
+        """Returns HTTPError with '404 not found' message"""
+        parent = self.get_parent_app()
+        if parent:
+            return parent.notfound()
+        else:
+            return web._NotFound()
+
+    def internalerror(self):
+        """Returns HTTPError with '500 internal error' message"""
+        parent = self.get_parent_app()
+        if parent:
+            return parent.internalerror()
+        elif web.config.get("debug"):
+            return debugerror()
+        else:
+            return web._InternalError()
+
+
+def with_metaclass(mcls):
+    def decorator(cls):
+        body = vars(cls).copy()
+        # clean out class body
+        body.pop("__dict__", None)
+        body.pop("__weakref__", None)
+        return mcls(cls.__name__, cls.__bases__, body)
+
+    return decorator
+
+
+class auto_application(application):
+    """Application similar to `application` but urls are constructed
+    automatically using metaclass.
+
+        >>> app = auto_application()
+        >>> class hello(app.page):
+        ...     def GET(self): return "hello, world"
+        ...
+        >>> class foo(app.page):
+        ...     path = '/foo/.*'
+        ...     def GET(self): return "foo"
+        >>> app.request("/hello").data
+        'hello, world'
+        >>> app.request('/foo/bar').data
+        'foo'
+    """
+
+    # PY3DOCTEST: b'hello, world'
+    # PY3DOCTEST: b'foo'
+
+    def __init__(self):
+        application.__init__(self)
+
+        class metapage(type):
+            def __init__(klass, name, bases, attrs):
+                type.__init__(klass, name, bases, attrs)
+                path = attrs.get("path", "/" + name)
+
+                # path can be specified as None to ignore that class
+                # typically required to create a abstract base class.
+                if path is not None:
+                    self.add_mapping(path, klass)
+
+        @with_metaclass(metapage)  # little hack needed for Py2 and Py3 compatibility
+        class page:
+            path = None
+
+        self.page = page
+
+
+# The application class already has the required functionality of subdir_application
+subdir_application = application
+
+
+class subdomain_application(application):
+    r"""
+    Application to delegate requests based on the host.
+
+        >>> urls = ("/hello", "hello")
+        >>> app = application(urls, globals())
+        >>> class hello:
+        ...     def GET(self): return "hello"
+        >>>
+        >>> mapping = (r"hello\.example\.com", app)
+        >>> app2 = subdomain_application(mapping)
+        >>> app2.request("/hello", host="hello.example.com").data
+        'hello'
+        >>> response = app2.request("/hello", host="something.example.com")
+        >>> response.status
+        '404 Not Found'
+        >>> response.data
+        'not found'
+    """
+
+    # PY3DOCTEST: b'hello'
+    # PY3DOCTEST: b'not found'
+
+    def handle(self):
+        host = web.ctx.host.split(":")[0]  # strip port
+        fn, args = self._match(self.mapping, host)
+        return self._delegate(fn, self.fvars, args)
+
+    def _match(self, mapping, value):
+        for pat, what in mapping:
+            if isinstance(what, str):
+                what, result = utils.re_subm("^" + pat + "$", what, value)
+            else:
+                result = utils.re_compile("^" + pat + "$").match(value)
+
+            if result:  # it's a match
+                return what, [x for x in result.groups()]
+        return None, None
+
+
+def loadhook(h):
+    """
+    Converts a load hook into an application processor.
+
+        >>> app = auto_application()
+        >>> def f(): "something done before handling request"
+        ...
+        >>> app.add_processor(loadhook(f))
+    """
+
+    def processor(handler):
+        h()
+        return handler()
+
+    return processor
+
+
+def unloadhook(h):
+    """
+    Converts an unload hook into an application processor.
+
+        >>> app = auto_application()
+        >>> def f(): "something done after handling request"
+        ...
+        >>> app.add_processor(unloadhook(f))
+    """
+
+    def processor(handler):
+        try:
+            result = handler()
+        except:
+            # run the hook even when handler raises some exception
+            h()
+            raise
+
+        if result and hasattr(result, "__next__"):
+            return wrap(result)
+        else:
+            h()
+            return result
+
+    def wrap(result):
+        def next_hook():
+            try:
+                return next(result)
+            except:
+                # call the hook at the and of iterator
+                h()
+                raise
+
+        result = iter(result)
+        while True:
+            try:
+                yield next_hook()
+            except StopIteration:
+                return
+
+    return processor
+
+
+def autodelegate(prefix=""):
+    """
+    Returns a method that takes one argument and calls the method named prefix+arg,
+    calling `notfound()` if there isn't one. Example:
+
+        urls = ('/prefs/(.*)', 'prefs')
+
+        class prefs:
+            GET = autodelegate('GET_')
+            def GET_password(self): pass
+            def GET_privacy(self): pass
+
+    `GET_password` would get called for `/prefs/password` while `GET_privacy` for
+    `GET_privacy` gets called for `/prefs/privacy`.
+
+    If a user visits `/prefs/password/change` then `GET_password(self, '/change')`
+    is called.
+    """
+
+    def internal(self, arg):
+        if "/" in arg:
+            first, rest = arg.split("/", 1)
+            func = prefix + first
+            args = ["/" + rest]
+        else:
+            func = prefix + arg
+            args = []
+
+        if hasattr(self, func):
+            try:
+                return getattr(self, func)(*args)
+            except TypeError:
+                raise web.notfound()
+        else:
+            raise web.notfound()
+
+    return internal
+
+
+class Reloader:
+    """Checks to see if any loaded modules have changed on disk and,
+    if so, reloads them.
+    """
+
+    """File suffix of compiled modules."""
+    if sys.platform.startswith("java"):
+        SUFFIX = "$py.class"
+    else:
+        SUFFIX = ".pyc"
+
+    def __init__(self):
+        self.mtimes = {}
+
+    def __call__(self):
+        sys_modules = list(sys.modules.values())
+        for mod in sys_modules:
+            self.check(mod)
+
+    def check(self, mod):
+        # jython registers java packages as modules but they either
+        # don't have a __file__ attribute or its value is None
+        if not (mod and hasattr(mod, "__file__") and mod.__file__):
+            return
+
+        try:
+            mtime = os.stat(mod.__file__).st_mtime
+        except OSError:
+            return
+        if mod.__file__.endswith(self.__class__.SUFFIX) and os.path.exists(
+            mod.__file__[:-1]
+        ):
+            mtime = max(os.stat(mod.__file__[:-1]).st_mtime, mtime)
+
+        if mod not in self.mtimes:
+            self.mtimes[mod] = mtime
+        elif self.mtimes[mod] < mtime:
+            try:
+                reload(mod)
+                self.mtimes[mod] = mtime
+            except ImportError:
+                pass
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

web/browser.py

@@ -0,0 +1,295 @@
+"""Browser to test web applications.
+(from web.py)
+"""
+import os
+import webbrowser
+from http.cookiejar import CookieJar
+from io import BytesIO
+from urllib.parse import urljoin
+from urllib.request import HTTPCookieProcessor, HTTPError, HTTPHandler, Request
+from urllib.request import build_opener as urllib_build_opener
+from urllib.response import addinfourl
+
+from .net import htmlunquote
+from .utils import re_compile
+
+DEBUG = False
+
+__all__ = ["BrowserError", "Browser", "AppBrowser", "AppHandler"]
+
+
+class BrowserError(Exception):
+    pass
+
+
+class Browser:
+    def __init__(self):
+        self.cookiejar = CookieJar()
+        self._cookie_processor = HTTPCookieProcessor(self.cookiejar)
+        self.form = None
+
+        self.url = "http://0.0.0.0:8080/"
+        self.path = "/"
+
+        self.status = None
+        self.data = None
+        self._response = None
+        self._forms = None
+
+    @property
+    def text(self):
+        return self.data.decode("utf-8")
+
+    def reset(self):
+        """Clears all cookies and history."""
+        self.cookiejar.clear()
+
+    def build_opener(self):
+        """Builds the opener using (urllib2/urllib.request).build_opener.
+        Subclasses can override this function to prodive custom openers.
+        """
+        return urllib_build_opener()
+
+    def do_request(self, req):
+        if DEBUG:
+            print("requesting", req.get_method(), req.get_full_url())
+
+        opener = self.build_opener()
+        opener.add_handler(self._cookie_processor)
+        try:
+            self._response = opener.open(req)
+        except HTTPError as e:
+            self._response = e
+
+        self.url = self._response.geturl()
+        self.path = Request(self.url).selector
+        self.data = self._response.read()
+        self.status = self._response.code
+        self._forms = None
+        self.form = None
+
+        return self.get_response()
+
+    def open(self, url, data=None, headers={}):
+        """Opens the specified url."""
+        url = urljoin(self.url, url)
+        req = Request(url, data, headers)
+
+        return self.do_request(req)
+
+    def show(self):
+        """Opens the current page in real web browser."""
+        f = open("page.html", "w")
+        f.write(self.data)
+        f.close()
+
+        url = "file://" + os.path.abspath("page.html")
+        webbrowser.open(url)
+
+    def get_response(self):
+        """Returns a copy of the current response."""
+        return addinfourl(
+            BytesIO(self.data), self._response.info(), self._response.geturl()
+        )
+
+    def get_soup(self):
+        """Returns beautiful soup of the current document."""
+        import BeautifulSoup
+
+        return BeautifulSoup.BeautifulSoup(self.data)
+
+    def get_text(self, e=None):
+        """Returns content of e or the current document as plain text."""
+        e = e or self.get_soup()
+        return "".join(
+            [htmlunquote(c) for c in e.recursiveChildGenerator() if isinstance(c, str)]
+        )
+
+    def _get_links(self):
+        soup = self.get_soup()
+        return [a for a in soup.findAll(name="a")]
+
+    def get_links(
+        self, text=None, text_regex=None, url=None, url_regex=None, predicate=None
+    ):
+        """Returns all links in the document."""
+        return self._filter_links(
+            self._get_links(),
+            text=text,
+            text_regex=text_regex,
+            url=url,
+            url_regex=url_regex,
+            predicate=predicate,
+        )
+
+    def follow_link(
+        self,
+        link=None,
+        text=None,
+        text_regex=None,
+        url=None,
+        url_regex=None,
+        predicate=None,
+    ):
+        if link is None:
+            links = self._filter_links(
+                self.get_links(),
+                text=text,
+                text_regex=text_regex,
+                url=url,
+                url_regex=url_regex,
+                predicate=predicate,
+            )
+            link = links and links[0]
+
+        if link:
+            return self.open(link["href"])
+        else:
+            raise BrowserError("No link found")
+
+    def find_link(
+        self, text=None, text_regex=None, url=None, url_regex=None, predicate=None
+    ):
+        links = self._filter_links(
+            self.get_links(),
+            text=text,
+            text_regex=text_regex,
+            url=url,
+            url_regex=url_regex,
+            predicate=predicate,
+        )
+        return links and links[0] or None
+
+    def _filter_links(
+        self,
+        links,
+        text=None,
+        text_regex=None,
+        url=None,
+        url_regex=None,
+        predicate=None,
+    ):
+        predicates = []
+        if text is not None:
+            predicates.append(lambda link: link.string == text)
+        if text_regex is not None:
+            predicates.append(
+                lambda link: re_compile(text_regex).search(link.string or "")
+            )
+        if url is not None:
+            predicates.append(lambda link: link.get("href") == url)
+        if url_regex is not None:
+            predicates.append(
+                lambda link: re_compile(url_regex).search(link.get("href", ""))
+            )
+        if predicate:
+            predicate.append(predicate)
+
+        def f(link):
+            for p in predicates:
+                if not p(link):
+                    return False
+            return True
+
+        return [link for link in links if f(link)]
+
+    def get_forms(self):
+        """Returns all forms in the current document.
+        The returned form objects implement the ClientForm.HTMLForm interface.
+        """
+        if self._forms is None:
+            import ClientForm
+
+            self._forms = ClientForm.ParseResponse(
+                self.get_response(), backwards_compat=False
+            )
+        return self._forms
+
+    def select_form(self, name=None, predicate=None, index=0):
+        """Selects the specified form."""
+        forms = self.get_forms()
+
+        if name is not None:
+            forms = [f for f in forms if f.name == name]
+        if predicate:
+            forms = [f for f in forms if predicate(f)]
+
+        if forms:
+            self.form = forms[index]
+            return self.form
+        else:
+            raise BrowserError("No form selected.")
+
+    def submit(self, **kw):
+        """submits the currently selected form."""
+        if self.form is None:
+            raise BrowserError("No form selected.")
+        req = self.form.click(**kw)
+        return self.do_request(req)
+
+    def __getitem__(self, key):
+        return self.form[key]
+
+    def __setitem__(self, key, value):
+        self.form[key] = value
+
+
+class AppBrowser(Browser):
+    """Browser interface to test web.py apps.
+
+    b = AppBrowser(app)
+    b.open('/')
+    b.follow_link(text='Login')
+
+    b.select_form(name='login')
+    b['username'] = 'joe'
+    b['password'] = 'secret'
+    b.submit()
+
+    assert b.path == '/'
+    assert 'Welcome joe' in b.get_text()
+    """
+
+    def __init__(self, app):
+        Browser.__init__(self)
+        self.app = app
+
+    def build_opener(self):
+        return urllib_build_opener(AppHandler(self.app))
+
+
+class AppHandler(HTTPHandler):
+    """urllib2 handler to handle requests using web.py application."""
+
+    handler_order = 100
+    https_request = HTTPHandler.do_request_
+
+    def __init__(self, app):
+        self.app = app
+
+    def http_open(self, req):
+        result = self.app.request(
+            localpart=req.selector,
+            method=req.get_method(),
+            host=req.host,
+            data=req.data,
+            headers=dict(req.header_items()),
+            https=(req.type == "https"),
+        )
+        return self._make_response(result, req.get_full_url())
+
+    def https_open(self, req):
+        return self.http_open(req)
+
+    def _make_response(self, result, url):
+
+        data = "\r\n".join([f"{k}: {v}" for k, v in result.header_items])
+
+        import email
+
+        headers = email.message_from_string(data)
+
+        response = addinfourl(BytesIO(result.data), headers, url)
+        code, msg = result.status.split(None, 1)
+        response.code, response.msg = int(code), msg
+        return response

web/contrib/template.py

@@ -0,0 +1,146 @@
+"""
+Interface to various templating engines.
+"""
+import os.path
+
+__all__ = ["render_cheetah", "render_genshi", "render_mako", "cache"]
+
+
+class render_cheetah:
+    """Rendering interface to Cheetah Templates.
+
+    Example:
+
+        render = render_cheetah('templates')
+        render.hello(name="cheetah")
+    """
+
+    def __init__(self, path):
+        # give error if Chetah is not installed
+        from Cheetah.Template import Template  # noqa: F401
+
+        self.path = path
+
+    def __getattr__(self, name):
+        from Cheetah.Template import Template
+
+        path = os.path.join(self.path, name + ".html")
+
+        def template(**kw):
+            t = Template(file=path, searchList=[kw])
+            return t.respond()
+
+        return template
+
+
+class render_genshi:
+    """Rendering interface genshi templates.
+    Example:
+
+    for xml/html templates.
+
+        render = render_genshi(['templates/'])
+        render.hello(name='genshi')
+
+    For text templates:
+
+        render = render_genshi(['templates/'], type='text')
+        render.hello(name='genshi')
+    """
+
+    def __init__(self, *a, **kwargs):
+        from genshi.template import TemplateLoader
+
+        self._type = kwargs.pop("type", None)
+        self._loader = TemplateLoader(*a, **kwargs)
+
+    def __getattr__(self, name):
+        # Assuming all templates are html
+        path = name + ".html"
+
+        if self._type == "text":
+            from genshi.template import TextTemplate
+
+            cls = TextTemplate
+            type = "text"
+        else:
+            cls = None
+            type = self._type
+
+        t = self._loader.load(path, cls=cls)
+
+        def template(**kw):
+            stream = t.generate(**kw)
+            if type:
+                return stream.render(type)
+            else:
+                return stream.render()
+
+        return template
+
+
+class render_jinja:
+    """Rendering interface to Jinja2 Templates
+
+    Example:
+
+        render= render_jinja('templates')
+        render.hello(name='jinja2')
+    """
+
+    def __init__(self, *a, **kwargs):
+        extensions = kwargs.pop("extensions", [])
+        globals = kwargs.pop("globals", {})
+
+        from jinja2 import Environment, FileSystemLoader
+
+        self._lookup = Environment(
+            loader=FileSystemLoader(*a, **kwargs), extensions=extensions
+        )
+        self._lookup.globals.update(globals)
+
+    def __getattr__(self, name):
+        # Assuming all templates end with .html
+        path = name + ".html"
+        t = self._lookup.get_template(path)
+        return t.render
+
+
+class render_mako:
+    """Rendering interface to Mako Templates.
+
+    Example:
+
+        render = render_mako(directories=['templates'])
+        render.hello(name="mako")
+    """
+
+    def __init__(self, *a, **kwargs):
+        from mako.lookup import TemplateLookup
+
+        self._lookup = TemplateLookup(*a, **kwargs)
+
+    def __getattr__(self, name):
+        # Assuming all templates are html
+        path = name + ".html"
+        t = self._lookup.get_template(path)
+        return t.render
+
+
+class cache:
+    """Cache for any rendering interface.
+
+    Example:
+
+        render = cache(render_cheetah("templates/"))
+        render.hello(name='cache')
+    """
+
+    def __init__(self, render):
+        self._render = render
+        self._cache = {}
+
+    def __getattr__(self, name):
+        if name not in self._cache:
+            self._cache[name] = getattr(self._render, name)
+        return self._cache[name]

web/debugerror.py

@@ -0,0 +1,377 @@
+"""
+pretty debug errors
+(part of web.py)
+
+portions adapted from Django <djangoproject.com>
+Copyright (c) 2005, the Lawrence Journal-World
+Used under the modified BSD license:
+http://www.xfree86.org/3.3.6/COPYRIGHT2.html#5
+"""
+
+__all__ = ["debugerror", "djangoerror", "emailerrors"]
+
+import os
+import os.path
+import pprint
+import sys
+import traceback
+
+from . import webapi as web
+from .net import websafe
+from .template import Template
+from .utils import safestr, sendmail
+
+
+def update_globals_template(t, globals):
+    t.t.__globals__.update(globals)
+
+
+whereami = os.path.join(os.getcwd(), __file__)
+whereami = os.path.sep.join(whereami.split(os.path.sep)[:-1])
+djangoerror_t = """\
+$def with (exception_type, exception_value, frames)
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html lang="en">
+<head>
+  <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+  <meta name="robots" content="NONE,NOARCHIVE" />
+  <title>$exception_type at $ctx.path</title>
+  <style type="text/css">
+    html * { padding:0; margin:0; }
+    body * { padding:10px 20px; }
+    body * * { padding:0; }
+    body { font:small sans-serif; }
+    body>div { border-bottom:1px solid #ddd; }
+    h1 { font-weight:normal; }
+    h2 { margin-bottom:.8em; }
+    h2 span { font-size:80%; color:#666; font-weight:normal; }
+    h3 { margin:1em 0 .5em 0; }
+    h4 { margin:0 0 .5em 0; font-weight: normal; }
+    table {
+        border:1px solid #ccc; border-collapse: collapse; background:white; }
+    tbody td, tbody th { vertical-align:top; padding:2px 3px; }
+    thead th {
+        padding:1px 6px 1px 3px; background:#fefefe; text-align:left;
+        font-weight:normal; font-size:11px; border:1px solid #ddd; }
+    tbody th { text-align:right; color:#666; padding-right:.5em; }
+    table.vars { margin:5px 0 2px 40px; }
+    table.vars td, table.req td { font-family:monospace; }
+    table td.code { width:100%;}
+    table td.code div { overflow:hidden; }
+    table.source th { color:#666; }
+    table.source td {
+        font-family:monospace; white-space:pre; border-bottom:1px solid #eee; }
+    ul.traceback { list-style-type:none; }
+    ul.traceback li.frame { margin-bottom:1em; }
+    div.context { margin: 10px 0; }
+    div.context ol {
+        padding-left:30px; margin:0 10px; list-style-position: inside; }
+    div.context ol li {
+        font-family:monospace; white-space:pre; color:#666; cursor:pointer; }
+    div.context ol.context-line li { color:black; background-color:#ccc; }
+    div.context ol.context-line li span { float: right; }
+    div.commands { margin-left: 40px; }
+    div.commands a { color:black; text-decoration:none; }
+    #summary { background: #ffc; }
+    #summary h2 { font-weight: normal; color: #666; }
+    #explanation { background:#eee; }
+    #template, #template-not-exist { background:#f6f6f6; }
+    #template-not-exist ul { margin: 0 0 0 20px; }
+    #traceback { background:#eee; }
+    #requestinfo { background:#f6f6f6; padding-left:120px; }
+    #summary table { border:none; background:transparent; }
+    #requestinfo h2, #requestinfo h3 { position:relative; margin-left:-100px; }
+    #requestinfo h3 { margin-bottom:-1em; }
+    .error { background: #ffc; }
+    .specific { color:#cc3300; font-weight:bold; }
+  </style>
+  <script type="text/javascript">
+  //<!--
+    function getElementsByClassName(oElm, strTagName, strClassName){
+        // Written by Jonathan Snook, http://www.snook.ca/jon;
+        // Add-ons by Robert Nyman, http://www.robertnyman.com
+        var arrElements = (strTagName == "*" && document.all)? document.all :
+        oElm.getElementsByTagName(strTagName);
+        var arrReturnElements = new Array();
+        strClassName = strClassName.replace(/\\-/g, "\\-");
+        var oRegExp = new RegExp("(^|\\s)" + strClassName + "(\\s|$$)");
+        var oElement;
+        for(var i=0; i<arrElements.length; i++){
+            oElement = arrElements[i];
+            if(oRegExp.test(oElement.className)){
+                arrReturnElements.push(oElement);
+            }
+        }
+        return (arrReturnElements)
+    }
+    function hideAll(elems) {
+      for (var e = 0; e < elems.length; e++) {
+        elems[e].style.display = 'none';
+      }
+    }
+    window.onload = function() {
+      hideAll(getElementsByClassName(document, 'table', 'vars'));
+      hideAll(getElementsByClassName(document, 'ol', 'pre-context'));
+      hideAll(getElementsByClassName(document, 'ol', 'post-context'));
+    }
+    function toggle() {
+      for (var i = 0; i < arguments.length; i++) {
+        var e = document.getElementById(arguments[i]);
+        if (e) {
+          e.style.display = e.style.display == 'none' ? 'block' : 'none';
+        }
+      }
+      return false;
+    }
+    function varToggle(link, id) {
+      toggle('v' + id);
+      var s = link.getElementsByTagName('span')[0];
+      var uarr = String.fromCharCode(0x25b6);
+      var darr = String.fromCharCode(0x25bc);
+      s.innerHTML = s.innerHTML == uarr ? darr : uarr;
+      return false;
+    }
+    //-->
+  </script>
+</head>
+<body>
+
+$def dicttable (d, kls='req', id=None):
+    $ items = d and list(d.items()) or []
+    $items.sort()
+    $:dicttable_items(items, kls, id)
+
+$def dicttable_items(items, kls='req', id=None):
+    $if items:
+        <table class="$kls"
+        $if id: id="$id"
+        ><thead><tr><th>Variable</th><th>Value</th></tr></thead>
+        <tbody>
+        $for k, v in items:
+            <tr><td>$k</td><td class="code"><div>$prettify(v)</div></td></tr>
+        </tbody>
+        </table>
+    $else:
+        <p>No data.</p>
+
+<div id="summary">
+  <h1>$exception_type at $ctx.path</h1>
+  <h2>$exception_value</h2>
+  <table><tr>
+    <th>Python</th>
+    <td>$frames[0].filename in $frames[0].function, line $frames[0].lineno</td>
+  </tr><tr>
+    <th>Web</th>
+    <td>$ctx.method $ctx.home$ctx.path</td>
+  </tr></table>
+</div>
+<div id="traceback">
+<h2>Traceback <span>(innermost first)</span></h2>
+<ul class="traceback">
+$for frame in frames:
+    <li class="frame">
+    <code>$frame.filename</code> in <code>$frame.function</code>
+    $if frame.context_line is not None:
+        <div class="context" id="c$frame.id">
+        $if frame.pre_context:
+            <ol start="$frame.pre_context_lineno" class="pre-context" id="pre$frame.id">
+            $for line in frame.pre_context:
+                <li onclick="toggle('pre$frame.id', 'post$frame.id')">$line</li>
+            </ol>
+            <ol start="$frame.lineno" class="context-line"><li onclick="toggle('pre$frame.id', 'post$frame.id')">$frame.context_line <span>...</span></li></ol>
+        $if frame.post_context:
+            <ol start='${frame.lineno + 1}' class="post-context" id="post$frame.id">
+            $for line in frame.post_context:
+                <li onclick="toggle('pre$frame.id', 'post$frame.id')">$line</li>
+            </ol>
+      </div>
+
+    $if frame.vars:
+        <div class="commands">
+        <a href='#' onclick="return varToggle(this, '$frame.id')"><span>&#x25b6;</span> Local vars</a>
+        $# $inspect.formatargvalues(*inspect.getargvalues(frame['tb'].tb_frame))
+        </div>
+        $:dicttable(frame.vars, kls='vars', id=('v' + str(frame.id)))
+      </li>
+  </ul>
+</div>
+
+<div id="requestinfo">
+$if ctx.output or ctx.headers:
+    <h2>Response so far</h2>
+    <h3>HEADERS</h3>
+    $:dicttable_items(ctx.headers)
+
+    <h3>BODY</h3>
+    <p class="req" style="padding-bottom: 2em"><code>
+    $ctx.output
+    </code></p>
+
+<h2>Request information</h2>
+
+<h3>INPUT</h3>
+$:dicttable(web.input(_unicode=False))
+
+<h3 id="cookie-info">COOKIES</h3>
+$:dicttable(web.cookies())
+
+<h3 id="meta-info">META</h3>
+$ newctx = [(k, v) for (k, v) in ctx.iteritems() if not k.startswith('_') and not isinstance(v, dict)]
+$:dicttable(dict(newctx))
+
+<h3 id="meta-info">ENVIRONMENT</h3>
+$:dicttable(ctx.env)
+</div>
+
+<div id="explanation">
+  <p>
+    You're seeing this error because you have <code>web.config.debug</code>
+    set to <code>True</code>. Set that to <code>False</code> if you don't want to see this.
+  </p>
+</div>
+
+</body>
+</html>
+"""  # noqa: W605
+
+djangoerror_r = None
+
+
+def djangoerror():
+    def _get_lines_from_file(filename, lineno, context_lines):
+        """
+        Returns context_lines before and after lineno from file.
+        Returns (pre_context_lineno, pre_context, context_line, post_context).
+        """
+        try:
+            source = open(filename).readlines()
+            lower_bound = max(0, lineno - context_lines)
+            upper_bound = lineno + context_lines
+
+            pre_context = [line.strip("\n") for line in source[lower_bound:lineno]]
+            context_line = source[lineno].strip("\n")
+            post_context = [
+                line.strip("\n") for line in source[lineno + 1 : upper_bound]
+            ]
+
+            return lower_bound, pre_context, context_line, post_context
+        except (OSError, IndexError):
+            return None, [], None, []
+
+    exception_type, exception_value, tback = sys.exc_info()
+    frames = []
+    while tback is not None:
+        filename = tback.tb_frame.f_code.co_filename
+        function = tback.tb_frame.f_code.co_name
+        lineno = tback.tb_lineno - 1
+
+        # hack to get correct line number for templates
+        lineno += tback.tb_frame.f_locals.get("__lineoffset__", 0)
+
+        (
+            pre_context_lineno,
+            pre_context,
+            context_line,
+            post_context,
+        ) = _get_lines_from_file(filename, lineno, 7)
+
+        if "__hidetraceback__" not in tback.tb_frame.f_locals:
+            frames.append(
+                web.storage(
+                    {
+                        "tback": tback,
+                        "filename": filename,
+                        "function": function,
+                        "lineno": lineno,
+                        "vars": tback.tb_frame.f_locals,
+                        "id": id(tback),
+                        "pre_context": pre_context,
+                        "context_line": context_line,
+                        "post_context": post_context,
+                        "pre_context_lineno": pre_context_lineno,
+                    }
+                )
+            )
+        tback = tback.tb_next
+    frames.reverse()
+
+    def prettify(x):
+        try:
+            out = pprint.pformat(x)
+        except Exception as e:
+            out = "[could not display: <" + e.__class__.__name__ + ": " + str(e) + ">]"
+        return out
+
+    global djangoerror_r
+    if djangoerror_r is None:
+        djangoerror_r = Template(djangoerror_t, filename=__file__, filter=websafe)
+
+    t = djangoerror_r
+    globals = {
+        "ctx": web.ctx,
+        "web": web,
+        "dict": dict,
+        "str": str,
+        "prettify": prettify,
+    }
+    update_globals_template(t, globals)
+    return t(exception_type, exception_value, frames)
+
+
+def debugerror():
+    """
+    A replacement for `internalerror` that presents a nice page with lots
+    of debug information for the programmer.
+
+    (Based on the beautiful 500 page from [Django](http://djangoproject.com/),
+    designed by [Wilson Miner](http://wilsonminer.com/).)
+    """
+    return web._InternalError(djangoerror())
+
+
+def emailerrors(to_address, olderror, from_address=None):
+    """
+    Wraps the old `internalerror` handler (pass as `olderror`) to
+    additionally email all errors to `to_address`, to aid in
+    debugging production websites.
+
+    Emails contain a normal text traceback as well as an
+    attachment containing the nice `debugerror` page.
+    """
+    from_address = from_address or to_address
+
+    def emailerrors_internal():
+        error = olderror()
+        tb = sys.exc_info()
+        error_name = tb[0]
+        error_value = tb[1]
+        tb_txt = "".join(traceback.format_exception(*tb))
+        path = web.ctx.path
+        request = web.ctx.method + " " + web.ctx.home + web.ctx.fullpath
+
+        message = f"\n{request}\n\n{tb_txt}\n\n"
+
+        sendmail(
+            "your buggy site <%s>" % from_address,
+            "the bugfixer <%s>" % to_address,
+            "bug: %(error_name)s: %(error_value)s (%(path)s)" % locals(),
+            message,
+            attachments=[dict(filename="bug.html", content=safestr(djangoerror()))],
+        )
+        return error
+
+    return emailerrors_internal
+
+
+if __name__ == "__main__":
+    urls = ("/", "index")
+    from .application import application
+
+    app = application(urls, globals())
+    app.internalerror = debugerror
+
+    class index:
+        def GET(self):
+            thisdoesnotexist  # noqa: F821
+
+    app.run()

web/form.py

@@ -0,0 +1,690 @@
+"""
+HTML forms
+(part of web.py)
+"""
+
+import copy
+import re
+
+from . import net, utils
+from . import webapi as web
+
+
+def attrget(obj, attr, value=None):
+    try:
+        if hasattr(obj, "has_key") and attr in obj:
+            return obj[attr]
+    except TypeError:
+        # Handle the case where has_key takes different number of arguments.
+        # This is the case with Model objects on appengine. See #134
+        pass
+    if (
+        hasattr(obj, "keys") and attr in obj
+    ):  # needed for Py3, has_key doesn't exist anymore
+        return obj[attr]
+    elif hasattr(obj, attr):
+        return getattr(obj, attr)
+    return value
+
+
+class Form:
+    r"""
+    HTML form.
+
+        >>> f = Form(Textbox("x"))
+        >>> f.render()
+        u'<table>\n    <tr><th><label for="x">x</label></th><td><input id="x" name="x" type="text"/></td></tr>\n</table>'
+        >>> f.fill(x="42")
+        True
+        >>> f.render()
+        u'<table>\n    <tr><th><label for="x">x</label></th><td><input id="x" name="x" type="text" value="42"/></td></tr>\n</table>'
+    """
+
+    def __init__(self, *inputs, **kw):
+        self.inputs = inputs
+        self.valid = True
+        self.note = None
+        self.validators = kw.pop("validators", [])
+
+    def __call__(self, x=None):
+        o = copy.deepcopy(self)
+        if x:
+            o.validates(x)
+        return o
+
+    def render(self):
+        out = ""
+        out += self.rendernote(self.note)
+        out += "<table>\n"
+
+        for i in self.inputs:
+            html = (
+                utils.safeunicode(i.pre)
+                + i.render()
+                + self.rendernote(i.note)
+                + utils.safeunicode(i.post)
+            )
+            if i.is_hidden():
+                out += '    <tr style="display: none;"><th></th><td>%s</td></tr>\n' % (
+                    html
+                )
+            else:
+                out += (
+                    '    <tr><th><label for="%s">%s</label></th><td>%s</td></tr>\n'
+                    % (net.websafe(i.id), net.websafe(i.description), html)
+                )
+        out += "</table>"
+        return out
+
+    def render_css(self):
+        out = []
+        out.append(self.rendernote(self.note))
+        for i in self.inputs:
+            if not i.is_hidden():
+                out.append(
+                    '<label for="%s">%s</label>'
+                    % (net.websafe(i.id), net.websafe(i.description))
+                )
+            out.append(i.pre)
+            out.append(i.render())
+            out.append(self.rendernote(i.note))
+            out.append(i.post)
+            out.append("\n")
+        return "".join(out)
+
+    def rendernote(self, note):
+        if note:
+            return '<strong class="wrong">%s</strong>' % net.websafe(note)
+        else:
+            return ""
+
+    def validates(self, source=None, _validate=True, **kw):
+        source = source or kw or web.input()
+        out = True
+        for i in self.inputs:
+            v = attrget(source, i.name)
+            if _validate:
+                out = i.validate(v) and out
+            else:
+                i.set_value(v)
+        if _validate:
+            out = out and self._validate(source)
+            self.valid = out
+        return out
+
+    def _validate(self, value):
+        self.value = value
+        for v in self.validators:
+            if not v.valid(value):
+                self.note = v.msg
+                return False
+        return True
+
+    def fill(self, source=None, **kw):
+        return self.validates(source, _validate=False, **kw)
+
+    def __getitem__(self, i):
+        for x in self.inputs:
+            if x.name == i:
+                return x
+        raise KeyError(i)
+
+    def __getattr__(self, name):
+        # don't interfere with deepcopy
+        inputs = self.__dict__.get("inputs") or []
+        for x in inputs:
+            if x.name == name:
+                return x
+        raise AttributeError(name)
+
+    def get(self, i, default=None):
+        try:
+            return self[i]
+        except KeyError:
+            return default
+
+    def _get_d(self):  # @@ should really be form.attr, no?
+        return utils.storage([(i.name, i.get_value()) for i in self.inputs])
+
+    d = property(_get_d)
+
+
+class Input:
+    """Generic input. Type attribute must be specified when called directly.
+
+    See also: <https://www.w3.org/TR/html52/sec-forms.html#the-input-element>
+
+    Currently only types which can be written inside one `<input />` tag are
+    supported.
+
+    - For checkbox, please use `Checkbox` class for better control.
+    - For radiobox, please use `Radio` class for better control.
+
+        >>> Input(name='foo', type='email', value="user@domain.com").render()
+        u'<input id="foo" name="foo" type="email" value="user@domain.com"/>'
+        >>> Input(name='foo', type='number', value="bar").render()
+        u'<input id="foo" name="foo" type="number" value="bar"/>'
+        >>> Input(name='num', type="number", min='0', max='10', step='2', value='5').render()
+        u'<input id="num" max="10" min="0" name="num" step="2" type="number" value="5"/>'
+        >>> Input(name='foo', type="tel", value='55512345').render()
+        u'<input id="foo" name="foo" type="tel" value="55512345"/>'
+        >>> Input(name='search', type="search", value='Search').render()
+        u'<input id="search" name="search" type="search" value="Search"/>'
+        >>> Input(name='search', type="search", value='Search', required='required', pattern='[a-z0-9]{2,30}', placeholder='Search...').render()
+        u'<input id="search" name="search" pattern="[a-z0-9]{2,30}" placeholder="Search..." required="required" type="search" value="Search"/>'
+        >>> Input(name='url', type="url", value='url').render()
+        u'<input id="url" name="url" type="url" value="url"/>'
+        >>> Input(name='range', type="range", min='0', max='10', step='2', value='5').render()
+        u'<input id="range" max="10" min="0" name="range" step="2" type="range" value="5"/>'
+        >>> Input(name='color', type="color").render()
+        u'<input id="color" name="color" type="color"/>'
+        >>> Input(name='f', type="file", accept=".doc,.docx,.xml").render()
+        u'<input accept=".doc,.docx,.xml" id="f" name="f" type="file"/>'
+    """
+
+    def __init__(self, name, *validators, **attrs):
+        self.name = name
+        self.validators = validators
+        self.attrs = attrs = AttributeList(attrs)
+
+        self.type = attrs.pop("type", None)
+        self.description = attrs.pop("description", name)
+        self.value = attrs.pop("value", None)
+        self.pre = attrs.pop("pre", "")
+        self.post = attrs.pop("post", "")
+        self.note = None
+
+        self.id = attrs.setdefault("id", self.get_default_id())
+
+        if "class_" in attrs:
+            attrs["class"] = attrs["class_"]
+            del attrs["class_"]
+
+    def is_hidden(self):
+        return False
+
+    def get_type(self):
+        if self.type is not None:
+            return self.type
+        else:
+            raise AttributeError("missing attribute 'type'")
+
+    def get_default_id(self):
+        return self.name
+
+    def validate(self, value):
+        self.set_value(value)
+
+        for v in self.validators:
+            if not v.valid(value):
+                self.note = v.msg
+                return False
+        return True
+
+    def set_value(self, value):
+        self.value = value
+
+    def get_value(self):
+        return self.value
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["type"] = self.get_type()
+        if self.value is not None:
+            attrs["value"] = self.value
+        attrs["name"] = self.name
+        attrs["id"] = self.id
+        return "<input %s/>" % attrs
+
+    def rendernote(self, note):
+        if note:
+            return '<strong class="wrong">%s</strong>' % net.websafe(note)
+        else:
+            return ""
+
+    def addatts(self):
+        # add leading space for backward-compatibility
+        return " " + str(self.attrs)
+
+
+class AttributeList(dict):
+    """List of attributes of input.
+
+    >>> a = AttributeList(type='text', name='x', value=20)
+    >>> a
+    <attrs: 'name="x" type="text" value="20"'>
+    """
+
+    def copy(self):
+        return AttributeList(self)
+
+    def __str__(self):
+        return " ".join([f'{k}="{net.websafe(v)}"' for k, v in sorted(self.items())])
+
+    def __repr__(self):
+        return "<attrs: %s>" % repr(str(self))
+
+
+class Textbox(Input):
+    """Textbox input.
+
+    >>> Textbox(name='foo', value='bar').render()
+    u'<input id="foo" name="foo" type="text" value="bar"/>'
+    >>> Textbox(name='foo', value=0).render()
+    u'<input id="foo" name="foo" type="text" value="0"/>'
+    """
+
+    def get_type(self):
+        return "text"
+
+
+class Password(Input):
+    """Password input.
+
+    >>> Password(name='password', value='secret').render()
+    u'<input id="password" name="password" type="password" value="secret"/>'
+    """
+
+    def get_type(self):
+        return "password"
+
+
+class Textarea(Input):
+    """Textarea input.
+
+    >>> Textarea(name='foo', value='bar').render()
+    u'<textarea id="foo" name="foo">bar</textarea>'
+    """
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["name"] = self.name
+        value = net.websafe(self.value or "")
+        return f"<textarea {attrs}>{value}</textarea>"
+
+
+class Dropdown(Input):
+    r"""Dropdown/select input.
+
+    >>> Dropdown(name='foo', args=['a', 'b', 'c'], value='b').render()
+    u'<select id="foo" name="foo">\n  <option value="a">a</option>\n  <option selected="selected" value="b">b</option>\n  <option value="c">c</option>\n</select>\n'
+    >>> Dropdown(name='foo', args=[('a', 'aa'), ('b', 'bb'), ('c', 'cc')], value='b').render()
+    u'<select id="foo" name="foo">\n  <option value="a">aa</option>\n  <option selected="selected" value="b">bb</option>\n  <option value="c">cc</option>\n</select>\n'
+    """
+
+    def __init__(self, name, args, *validators, **attrs):
+        self.args = args
+        super().__init__(name, *validators, **attrs)
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["name"] = self.name
+
+        x = "<select %s>\n" % attrs
+
+        for arg in self.args:
+            x += self._render_option(arg)
+
+        x += "</select>\n"
+        return x
+
+    def _render_option(self, arg, indent="  "):
+        if isinstance(arg, (tuple, list)):
+            value, desc = arg
+        else:
+            value, desc = arg, arg
+
+        value = utils.safestr(value)
+        if isinstance(self.value, (tuple, list)):
+            s_value = [utils.safestr(x) for x in self.value]
+        else:
+            s_value = utils.safestr(self.value)
+
+        if s_value == value or (isinstance(s_value, list) and value in s_value):
+            select_p = ' selected="selected"'
+        else:
+            select_p = ""
+        return indent + '<option{} value="{}">{}</option>\n'.format(
+            select_p,
+            net.websafe(value),
+            net.websafe(desc),
+        )
+
+
+class GroupedDropdown(Dropdown):
+    r"""Grouped Dropdown/select input.
+
+    >>> GroupedDropdown(name='car_type', args=(('Swedish Cars', ('Volvo', 'Saab')), ('German Cars', ('Mercedes', 'Audi'))), value='Audi').render()
+    u'<select id="car_type" name="car_type">\n  <optgroup label="Swedish Cars">\n    <option value="Volvo">Volvo</option>\n    <option value="Saab">Saab</option>\n  </optgroup>\n  <optgroup label="German Cars">\n    <option value="Mercedes">Mercedes</option>\n    <option selected="selected" value="Audi">Audi</option>\n  </optgroup>\n</select>\n'
+    >>> GroupedDropdown(name='car_type', args=(('Swedish Cars', (('v', 'Volvo'), ('s', 'Saab'))), ('German Cars', (('m', 'Mercedes'), ('a', 'Audi')))), value='a').render()
+    u'<select id="car_type" name="car_type">\n  <optgroup label="Swedish Cars">\n    <option value="v">Volvo</option>\n    <option value="s">Saab</option>\n  </optgroup>\n  <optgroup label="German Cars">\n    <option value="m">Mercedes</option>\n    <option selected="selected" value="a">Audi</option>\n  </optgroup>\n</select>\n'
+
+    """
+
+    def __init__(self, name, args, *validators, **attrs):
+        self.args = args
+        super().__init__(name, *validators, **attrs)
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["name"] = self.name
+
+        x = "<select %s>\n" % attrs
+
+        for label, options in self.args:
+            x += '  <optgroup label="%s">\n' % net.websafe(label)
+            for arg in options:
+                x += self._render_option(arg, indent="    ")
+            x += "  </optgroup>\n"
+
+        x += "</select>\n"
+        return x
+
+
+class Radio(Input):
+    def __init__(self, name, args, *validators, **attrs):
+        self.args = args
+        super().__init__(name, *validators, **attrs)
+
+    def render(self):
+        x = "<span>"
+        for idx, arg in enumerate(self.args, start=1):
+            if isinstance(arg, (tuple, list)):
+                value, desc = arg
+            else:
+                value, desc = arg, arg
+            attrs = self.attrs.copy()
+            attrs["name"] = self.name
+            attrs["type"] = "radio"
+            attrs["value"] = value
+            attrs["id"] = self.name + str(idx)
+            if self.value == value:
+                attrs["checked"] = "checked"
+            x += f"<input {attrs}/> {net.websafe(desc)}"
+        x += "</span>"
+        return x
+
+
+class Checkbox(Input):
+    """Checkbox input.
+
+    >>> Checkbox('foo', value='bar', checked=True).render()
+    u'<input checked="checked" id="foo_bar" name="foo" type="checkbox" value="bar"/>'
+    >>> Checkbox('foo', value='bar').render()
+    u'<input id="foo_bar" name="foo" type="checkbox" value="bar"/>'
+    >>> c = Checkbox('foo', value='bar')
+    >>> c.validate('on')
+    True
+    >>> c.render()
+    u'<input checked="checked" id="foo_bar" name="foo" type="checkbox" value="bar"/>'
+    """
+
+    def __init__(self, name, *validators, **attrs):
+        self.checked = attrs.pop("checked", False)
+        Input.__init__(self, name, *validators, **attrs)
+
+    def get_default_id(self):
+        value = utils.safestr(self.value or "")
+        return self.name + "_" + value.replace(" ", "_")
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["type"] = "checkbox"
+        attrs["name"] = self.name
+        attrs["value"] = self.value
+
+        if self.checked:
+            attrs["checked"] = "checked"
+        return "<input %s/>" % attrs
+
+    def set_value(self, value):
+        self.checked = bool(value)
+
+    def get_value(self):
+        return self.checked
+
+
+class Button(Input):
+    """HTML Button.
+
+    >>> Button("save").render()
+    u'<button id="save" name="save">save</button>'
+    >>> Button("action", value="save", html="<b>Save Changes</b>").render()
+    u'<button id="action" name="action" value="save"><b>Save Changes</b></button>'
+    """
+
+    def __init__(self, name, *validators, **attrs):
+        super().__init__(name, *validators, **attrs)
+        self.description = ""
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["name"] = self.name
+        if self.value is not None:
+            attrs["value"] = self.value
+        html = attrs.pop("html", None) or net.websafe(self.name)
+        return f"<button {attrs}>{html}</button>"
+
+
+class Hidden(Input):
+    """Hidden Input.
+
+    >>> Hidden(name='foo', value='bar').render()
+    u'<input id="foo" name="foo" type="hidden" value="bar"/>'
+    """
+
+    def is_hidden(self):
+        return True
+
+    def get_type(self):
+        return "hidden"
+
+
+class File(Input):
+    """File input.
+
+    >>> File(name='f', accept=".doc,.docx,.xml").render()
+    u'<input accept=".doc,.docx,.xml" id="f" name="f" type="file"/>'
+    """
+
+    def get_type(self):
+        return "file"
+
+
+class Telephone(Input):
+    """Telephone input.
+
+    See: <https://html.spec.whatwg.org/#telephone-state-(type=tel)>
+
+    >>> Telephone(name='tel', value='55512345').render()
+    u'<input id="tel" name="tel" type="tel" value="55512345"/>'
+    """
+
+    def get_type(self):
+        return "tel"
+
+
+class Email(Input):
+    """Email input.
+
+    See: <https://html.spec.whatwg.org/#e-mail-state-(type=email)>
+
+    >>> Email(name='email', value='me@example.org').render()
+    u'<input id="email" name="email" type="email" value="me@example.org"/>'
+
+    """
+
+    def get_type(self):
+        return "email"
+
+
+class Date(Input):
+    """Date input.
+
+    Note: Not supported by desktop Safari, Internet Explorer, or Opera Mini
+
+    See: <https://html.spec.whatwg.org/#date-state-(type=date)>
+
+    >>> Date(name='date', value='2020-04-01').render()
+    u'<input id="date" name="date" type="date" value="2020-04-01"/>'
+
+    """
+
+    def get_type(self):
+        return "date"
+
+
+class Time(Input):
+    """Time input.
+
+    Note: Not supported by desktop Safari, Internet Explorer, or Opera Mini
+
+    See: <https://html.spec.whatwg.org/#time-state-(type=time)>
+
+    >>> Time(name='time', value='07:00').render()
+    u'<input id="time" name="time" type="time" value="07:00"/>'
+
+    """
+
+    def get_type(self):
+        return "time"
+
+
+class Search(Input):
+    """Search input.
+
+    See: <https://html.spec.whatwg.org/#text-(type=text)-state-and-search-state-(type=search)>
+
+    >> Search(name='search', value='Search').render()
+    u'<input id="search" name="search" type="search" value="Search"/>'
+    >>> Search(name='search', value='Search', required='required', pattern='[a-z0-9]{2,30}', placeholder='Search...').render()
+    u'<input id="search" name="search" pattern="[a-z0-9]{2,30}" placeholder="Search..." required="required" type="search" value="Search"/>'
+
+    """
+
+    def get_type(self):
+        return "search"
+
+
+class Url(Input):
+    """URL input.
+
+    See: <https://html.spec.whatwg.org/#url-state-(type=url)>
+
+    >>> Url(name='url', value='url').render()
+    u'<input id="url" name="url" type="url" value="url"/>'
+    """
+
+    def get_type(self):
+        return "url"
+
+
+class Number(Input):
+    """Number input.
+
+    See: <https://html.spec.whatwg.org/#number-state-(type=number)>
+
+    >>> Number(name='num', min='0', max='10', step='2', value='5').render()
+    u'<input id="num" max="10" min="0" name="num" step="2" type="number" value="5"/>'
+    """
+
+    def get_type(self):
+        return "number"
+
+
+class Range(Input):
+    """Range input.
+
+    See: <https://html.spec.whatwg.org/#range-state-(type=range)>
+
+    >>> Range(name='range', min='0', max='10', step='2', value='5').render()
+    u'<input id="range" max="10" min="0" name="range" step="2" type="range" value="5"/>'
+    """
+
+    def get_type(self):
+        return "range"
+
+
+class Color(Input):
+    """Color input.
+
+    Note: Not supported by Internet Explorer or Opera Mini
+
+    See: <https://html.spec.whatwg.org/#color-stat://html.spec.whatwg.org/#color-state-(type=color)>
+
+    >>> Color(name='color').render()
+    u'<input id="color" name="color" type="color"/>'
+    """
+
+    def get_type(self):
+        return "color"
+
+
+class Datalist(Input):
+    """Datalist input.
+
+    This is currently supported by Chrome, Firefox, Edge, and Opera. It is not
+    supported on Safari or Internet Explorer. Use it with caution.
+
+    Datalist cannot be used separately. It must be bound to an input.
+
+    <https://html.spec.whatwg.org/#the-datalist-element>
+
+    >>> Datalist(name='list', args=[('a', 'b'), ('c', 'd')]).render()
+    u'<datalist id="list" name="list"><option label="a" value="b"/><option label="c" value="d"/></datalist>'
+    >>> Datalist(name='list', args=[['a', 'b'], ['c', 'd']]).render()
+    u'<datalist id="list" name="list"><option label="a" value="b"/><option label="c" value="d"/></datalist>'
+    >>> Datalist(name='list', args=['a', 'b', 'c', 'd']).render()
+    u'<datalist id="list" name="list"><option value="a"/><option value="b"/><option value="c"/><option value="d"/></datalist>'
+    """
+
+    def __init__(self, name, args, *validators, **kwargs):
+        self.args = args
+        super().__init__(name, *validators, **kwargs)
+
+    def render(self):
+        attrs = self.attrs.copy()
+        attrs["name"] = self.name
+        label_p = ""
+        x = "<datalist %s>" % attrs
+        for arg in self.args:
+            if isinstance(arg, (tuple, list)):
+                label_p = ' label="%s"' % net.websafe(arg[0])
+                label = net.websafe(arg[1])
+            else:
+                label = net.websafe(arg)
+            x += f'<option{label_p} value="{label}"/>'
+        x += "</datalist>"
+        return x
+
+
+class Validator:
+    def __deepcopy__(self, memo):
+        return copy.copy(self)
+
+    def __init__(self, msg, test, jstest=None):
+        utils.autoassign(self, locals())
+
+    def valid(self, value):
+        try:
+            return self.test(value)
+        except:
+            return False
+
+
+notnull = Validator("Required", bool)
+
+
+class regexp(Validator):
+    def __init__(self, rexp, msg):
+        self.rexp = re.compile(rexp)
+        self.msg = msg
+
+    def valid(self, value):
+        return bool(self.rexp.match(value))
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

web/http.py

@@ -0,0 +1,168 @@
+"""
+HTTP Utilities
+(from web.py)
+"""
+
+__all__ = [
+    "expires",
+    "lastmodified",
+    "prefixurl",
+    "modified",
+    "changequery",
+    "url",
+    "profiler",
+]
+
+import datetime
+
+from . import net, utils
+from . import webapi as web
+from .py3helpers import iteritems
+
+try:
+    from urllib.parse import urlencode as urllib_urlencode
+except ImportError:
+    from urllib import urlencode as urllib_urlencode
+
+
+def prefixurl(base=""):
+    """
+    Sorry, this function is really difficult to explain.
+    Maybe some other time.
+    """
+    url = web.ctx.path.lstrip("/")
+    for i in range(url.count("/")):
+        base += "../"
+    if not base:
+        base = "./"
+    return base
+
+
+def expires(delta):
+    """
+    Outputs an `Expires` header for `delta` from now.
+    `delta` is a `timedelta` object or a number of seconds.
+    """
+    if isinstance(delta, int):
+        delta = datetime.timedelta(seconds=delta)
+    date_obj = datetime.datetime.utcnow() + delta
+    web.header("Expires", net.httpdate(date_obj))
+
+
+def lastmodified(date_obj):
+    """Outputs a `Last-Modified` header for `datetime`."""
+    web.header("Last-Modified", net.httpdate(date_obj))
+
+
+def modified(date=None, etag=None):
+    """
+    Checks to see if the page has been modified since the version in the
+    requester's cache.
+
+    When you publish pages, you can include `Last-Modified` and `ETag`
+    with the date the page was last modified and an opaque token for
+    the particular version, respectively. When readers reload the page,
+    the browser sends along the modification date and etag value for
+    the version it has in its cache. If the page hasn't changed,
+    the server can just return `304 Not Modified` and not have to
+    send the whole page again.
+
+    This function takes the last-modified date `date` and the ETag `etag`
+    and checks the headers to see if they match. If they do, it returns
+    `True`, or otherwise it raises NotModified error. It also sets
+    `Last-Modified` and `ETag` output headers.
+    """
+    n = {x.strip('" ') for x in web.ctx.env.get("HTTP_IF_NONE_MATCH", "").split(",")}
+    m = net.parsehttpdate(web.ctx.env.get("HTTP_IF_MODIFIED_SINCE", "").split(";")[0])
+    validate = False
+    if etag:
+        if "*" in n or etag in n:
+            validate = True
+    if date and m:
+        # we subtract a second because
+        # HTTP dates don't have sub-second precision
+        if date - datetime.timedelta(seconds=1) <= m:
+            validate = True
+
+    if date:
+        lastmodified(date)
+    if etag:
+        web.header("ETag", '"' + etag + '"')
+    if validate:
+        raise web.notmodified()
+    else:
+        return True
+
+
+def urlencode(query, doseq=0):
+    """
+    Same as urllib.urlencode, but supports unicode strings.
+
+        >>> urlencode({'text':'foo bar'})
+        'text=foo+bar'
+        >>> urlencode({'x': [1, 2]}, doseq=True)
+        'x=1&x=2'
+    """
+
+    def convert(value, doseq=False):
+        if doseq and isinstance(value, list):
+            return [convert(v) for v in value]
+        else:
+            return utils.safestr(value)
+
+    query = {k: convert(v, doseq) for k, v in query.items()}
+    return urllib_urlencode(query, doseq=doseq)
+
+
+def changequery(query=None, **kw):
+    """
+    Imagine you're at `/foo?a=1&b=2`. Then `changequery(a=3)` will return
+    `/foo?a=3&b=2` -- the same URL but with the arguments you requested
+    changed.
+    """
+    if query is None:
+        query = web.rawinput(method="get")
+    for k, v in iteritems(kw):
+        if v is None:
+            query.pop(k, None)
+        else:
+            query[k] = v
+    out = web.ctx.path
+    if query:
+        out += "?" + urlencode(query, doseq=True)
+    return out
+
+
+def url(path=None, doseq=False, **kw):
+    """
+    Makes url by concatenating web.ctx.homepath and path and the
+    query string created using the arguments.
+    """
+    if path is None:
+        path = web.ctx.path
+    if path.startswith("/"):
+        out = web.ctx.homepath + path
+    else:
+        out = path
+
+    if kw:
+        out += "?" + urlencode(kw, doseq=doseq)
+
+    return out
+
+
+def profiler(app):
+    """Outputs basic profiling information at the bottom of each response."""
+    from utils import profile
+
+    def profile_internal(e, o):
+        out, result = profile(app)(e, o)
+        return list(out) + ["<pre>" + net.websafe(result) + "</pre>"]
+
+    return profile_internal
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

web/httpserver.py

@@ -0,0 +1,306 @@
+import os
+import posixpath
+import sys
+from http.server import BaseHTTPRequestHandler, HTTPServer, SimpleHTTPRequestHandler
+from io import BytesIO
+from urllib import parse as urlparse
+from urllib.parse import unquote
+
+from . import utils
+from . import webapi as web
+
+__all__ = ["runsimple"]
+
+
+def runbasic(func, server_address=("0.0.0.0", 8080)):
+    """
+      Runs a simple HTTP server hosting WSGI app `func`. The directory `static/`
+      is hosted statically.
+
+      Based on [WsgiServer][ws] from [Colin Stewart][cs].
+
+    [ws]: http://www.owlfish.com/software/wsgiutils/documentation/wsgi-server-api.html
+    [cs]: http://www.owlfish.com/
+    """
+    # Copyright (c) 2004 Colin Stewart (http://www.owlfish.com/)
+    # Modified somewhat for simplicity
+    # Used under the modified BSD license:
+    # http://www.xfree86.org/3.3.6/COPYRIGHT2.html#5
+
+    import errno
+    import socket
+    import traceback
+
+    import SocketServer
+
+    class WSGIHandler(SimpleHTTPRequestHandler):
+        def run_wsgi_app(self):
+            protocol, host, path, parameters, query, fragment = urlparse.urlparse(
+                "http://dummyhost%s" % self.path
+            )
+
+            # we only use path, query
+            env = {
+                "wsgi.version": (1, 0),
+                "wsgi.url_scheme": "http",
+                "wsgi.input": self.rfile,
+                "wsgi.errors": sys.stderr,
+                "wsgi.multithread": 1,
+                "wsgi.multiprocess": 0,
+                "wsgi.run_once": 0,
+                "REQUEST_METHOD": self.command,
+                "REQUEST_URI": self.path,
+                "PATH_INFO": path,
+                "QUERY_STRING": query,
+                "CONTENT_TYPE": self.headers.get("Content-Type", ""),
+                "CONTENT_LENGTH": self.headers.get("Content-Length", ""),
+                "REMOTE_ADDR": self.client_address[0],
+                "SERVER_NAME": self.server.server_address[0],
+                "SERVER_PORT": str(self.server.server_address[1]),
+                "SERVER_PROTOCOL": self.request_version,
+            }
+
+            for http_header, http_value in self.headers.items():
+                env["HTTP_%s" % http_header.replace("-", "_").upper()] = http_value
+
+            # Setup the state
+            self.wsgi_sent_headers = 0
+            self.wsgi_headers = []
+
+            try:
+                # We have there environment, now invoke the application
+                result = self.server.app(env, self.wsgi_start_response)
+                try:
+                    try:
+                        for data in result:
+                            if data:
+                                self.wsgi_write_data(data)
+                    finally:
+                        if hasattr(result, "close"):
+                            result.close()
+                except OSError as socket_err:
+                    # Catch common network errors and suppress them
+                    if socket_err.args[0] in (errno.ECONNABORTED, errno.EPIPE):
+                        return
+                except socket.timeout:
+                    return
+            except:
+                print(traceback.format_exc(), file=web.debug)
+
+            if not self.wsgi_sent_headers:
+                # We must write out something!
+                self.wsgi_write_data(" ")
+            return
+
+        do_POST = run_wsgi_app
+        do_PUT = run_wsgi_app
+        do_DELETE = run_wsgi_app
+
+        def do_GET(self):
+            if self.path.startswith("/static/"):
+                SimpleHTTPRequestHandler.do_GET(self)
+            else:
+                self.run_wsgi_app()
+
+        def wsgi_start_response(self, response_status, response_headers, exc_info=None):
+            if self.wsgi_sent_headers:
+                raise Exception("Headers already sent and start_response called again!")
+            # Should really take a copy to avoid changes in the application....
+            self.wsgi_headers = (response_status, response_headers)
+            return self.wsgi_write_data
+
+        def wsgi_write_data(self, data):
+            if not self.wsgi_sent_headers:
+                status, headers = self.wsgi_headers
+                # Need to send header prior to data
+                status_code = status[: status.find(" ")]
+                status_msg = status[status.find(" ") + 1 :]
+                self.send_response(int(status_code), status_msg)
+                for header, value in headers:
+                    self.send_header(header, value)
+                self.end_headers()
+                self.wsgi_sent_headers = 1
+            # Send the data
+            self.wfile.write(data)
+
+    class WSGIServer(SocketServer.ThreadingMixIn, HTTPServer):
+        def __init__(self, func, server_address):
+            HTTPServer.HTTPServer.__init__(self, server_address, WSGIHandler)
+            self.app = func
+            self.serverShuttingDown = 0
+
+    print("http://%s:%d/" % server_address)
+    WSGIServer(func, server_address).serve_forever()
+
+
+# The WSGIServer instance.
+# Made global so that it can be stopped in embedded mode.
+server = None
+
+
+def runsimple(func, server_address=("0.0.0.0", 8080)):
+    """
+    Runs [CherryPy][cp] WSGI server hosting WSGI app `func`.
+    The directory `static/` is hosted statically.
+
+    [cp]: http://www.cherrypy.org
+    """
+    global server
+    func = StaticMiddleware(func)
+    func = LogMiddleware(func)
+
+    server = WSGIServer(server_address, func)
+
+    if "/" in server_address[0]:
+        print("%s" % server_address)
+    else:
+        if server.ssl_adapter:
+            print("https://%s:%d/" % server_address)
+        else:
+            print("http://%s:%d/" % server_address)
+
+    try:
+        server.start()
+    except (KeyboardInterrupt, SystemExit):
+        server.stop()
+        server = None
+
+
+def WSGIServer(server_address, wsgi_app):
+    """Creates CherryPy WSGI server listening at `server_address` to serve `wsgi_app`.
+    This function can be overwritten to customize the webserver or use a different webserver.
+    """
+    from cheroot import wsgi
+
+    server = wsgi.Server(server_address, wsgi_app, server_name="localhost")
+    server.nodelay = not sys.platform.startswith(
+        "java"
+    )  # TCP_NODELAY isn't supported on the JVM
+    return server
+
+
+class StaticApp(SimpleHTTPRequestHandler):
+    """WSGI application for serving static files."""
+
+    def __init__(self, environ, start_response):
+        self.headers = []
+        self.environ = environ
+        self.start_response = start_response
+        self.directory = os.getcwd()
+
+    def send_response(self, status, msg=""):
+        # the int(status) call is needed because in Py3 status is an enum.IntEnum and we need the integer behind
+        self.status = str(int(status)) + " " + msg
+
+    def send_header(self, name, value):
+        self.headers.append((name, value))
+
+    def end_headers(self):
+        pass
+
+    def log_message(*a):
+        pass
+
+    def __iter__(self):
+        environ = self.environ
+
+        self.path = environ.get("PATH_INFO", "")
+        self.client_address = (
+            environ.get("REMOTE_ADDR", "-"),
+            environ.get("REMOTE_PORT", "-"),
+        )
+        self.command = environ.get("REQUEST_METHOD", "-")
+
+        self.wfile = BytesIO()  # for capturing error
+
+        try:
+            path = self.translate_path(self.path)
+            etag = '"%s"' % os.path.getmtime(path)
+            client_etag = environ.get("HTTP_IF_NONE_MATCH")
+            self.send_header("ETag", etag)
+            if etag == client_etag:
+                self.send_response(304, "Not Modified")
+                self.start_response(self.status, self.headers)
+                return
+        except OSError:
+            pass  # Probably a 404
+
+        f = self.send_head()
+        self.start_response(self.status, self.headers)
+
+        if f:
+            block_size = 16 * 1024
+            while True:
+                buf = f.read(block_size)
+                if not buf:
+                    break
+                yield buf
+            f.close()
+        else:
+            value = self.wfile.getvalue()
+            yield value
+
+
+class StaticMiddleware:
+    """WSGI middleware for serving static files."""
+
+    def __init__(self, app, prefix="/static/"):
+        self.app = app
+        self.prefix = prefix
+
+    def __call__(self, environ, start_response):
+        path = environ.get("PATH_INFO", "")
+        path = self.normpath(path)
+
+        if path.startswith(self.prefix):
+            return StaticApp(environ, start_response)
+        else:
+            return self.app(environ, start_response)
+
+    def normpath(self, path):
+        path2 = posixpath.normpath(unquote(path))
+        if path.endswith("/"):
+            path2 += "/"
+        return path2
+
+
+class LogMiddleware:
+    """WSGI middleware for logging the status."""
+
+    def __init__(self, app):
+        self.app = app
+        self.format = '%s - - [%s] "%s %s %s" - %s'
+
+        f = BytesIO()
+
+        class FakeSocket:
+            def makefile(self, *a):
+                return f
+
+        # take log_date_time_string method from BaseHTTPRequestHandler
+        self.log_date_time_string = BaseHTTPRequestHandler(
+            FakeSocket(), None, None
+        ).log_date_time_string
+
+    def __call__(self, environ, start_response):
+        def xstart_response(status, response_headers, *args):
+            out = start_response(status, response_headers, *args)
+            self.log(status, environ)
+            return out
+
+        return self.app(environ, xstart_response)
+
+    def log(self, status, environ):
+        outfile = environ.get("wsgi.errors", web.debug)
+        req = environ.get("PATH_INFO", "_")
+        protocol = environ.get("ACTUAL_SERVER_PROTOCOL", "-")
+        method = environ.get("REQUEST_METHOD", "-")
+        host = "{}:{}".format(
+            environ.get("REMOTE_ADDR", "-"),
+            environ.get("REMOTE_PORT", "-"),
+        )
+
+        time = self.log_date_time_string()
+
+        msg = self.format % (host, time, protocol, method, req, status)
+        print(utils.safestr(msg), file=outfile)

web/net.py

@@ -0,0 +1,279 @@
+"""
+Network Utilities
+(from web.py)
+"""
+
+
+import datetime
+import re
+import socket
+import time
+
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
+
+__all__ = [
+    "validipaddr",
+    "validip6addr",
+    "validipport",
+    "validip",
+    "validaddr",
+    "urlquote",
+    "httpdate",
+    "parsehttpdate",
+    "htmlquote",
+    "htmlunquote",
+    "websafe",
+]
+
+
+def validip6addr(address):
+    """
+    Returns True if `address` is a valid IPv6 address.
+
+        >>> validip6addr('::')
+        True
+        >>> validip6addr('aaaa:bbbb:cccc:dddd::1')
+        True
+        >>> validip6addr('1:2:3:4:5:6:7:8:9:10')
+        False
+        >>> validip6addr('12:10')
+        False
+    """
+    try:
+        socket.inet_pton(socket.AF_INET6, address)
+    except (OSError, AttributeError, ValueError):
+        return False
+
+    return True
+
+
+def validipaddr(address):
+    """
+    Returns True if `address` is a valid IPv4 address.
+
+        >>> validipaddr('192.168.1.1')
+        True
+        >>> validipaddr('192.168. 1.1')
+        False
+        >>> validipaddr('192.168.1.800')
+        False
+        >>> validipaddr('192.168.1')
+        False
+    """
+    try:
+        octets = address.split(".")
+        if len(octets) != 4:
+            return False
+
+        for x in octets:
+            if " " in x:
+                return False
+
+            if not (0 <= int(x) <= 255):
+                return False
+    except ValueError:
+        return False
+    return True
+
+
+def validipport(port):
+    """
+    Returns True if `port` is a valid IPv4 port.
+
+        >>> validipport('9000')
+        True
+        >>> validipport('foo')
+        False
+        >>> validipport('1000000')
+        False
+    """
+    try:
+        if not (0 <= int(port) <= 65535):
+            return False
+    except ValueError:
+        return False
+    return True
+
+
+def validip(ip, defaultaddr="0.0.0.0", defaultport=8080):
+    """
+    Returns `(ip_address, port)` from string `ip_addr_port`
+
+        >>> validip('1.2.3.4')
+        ('1.2.3.4', 8080)
+        >>> validip('80')
+        ('0.0.0.0', 80)
+        >>> validip('192.168.0.1:85')
+        ('192.168.0.1', 85)
+        >>> validip('::')
+        ('::', 8080)
+        >>> validip('[::]:88')
+        ('::', 88)
+        >>> validip('[::1]:80')
+        ('::1', 80)
+
+    """
+    addr = defaultaddr
+    port = defaultport
+
+    # Matt Boswell's code to check for ipv6 first
+    match = re.search(r"^\[([^]]+)\](?::(\d+))?$", ip)  # check for [ipv6]:port
+    if match:
+        if validip6addr(match.group(1)):
+            if match.group(2):
+                if validipport(match.group(2)):
+                    return (match.group(1), int(match.group(2)))
+            else:
+                return (match.group(1), port)
+    else:
+        if validip6addr(ip):
+            return (ip, port)
+    # end ipv6 code
+
+    ip = ip.split(":", 1)
+    if len(ip) == 1:
+        if not ip[0]:
+            pass
+        elif validipaddr(ip[0]):
+            addr = ip[0]
+        elif validipport(ip[0]):
+            port = int(ip[0])
+        else:
+            raise ValueError(":".join(ip) + " is not a valid IP address/port")
+    elif len(ip) == 2:
+        addr, port = ip
+        if not validipaddr(addr) or not validipport(port):
+            raise ValueError(":".join(ip) + " is not a valid IP address/port")
+        port = int(port)
+    else:
+        raise ValueError(":".join(ip) + " is not a valid IP address/port")
+    return (addr, port)
+
+
+def validaddr(string_):
+    """
+    Returns either (ip_address, port) or "/path/to/socket" from string_
+
+        >>> validaddr('/path/to/socket')
+        '/path/to/socket'
+        >>> validaddr('8000')
+        ('0.0.0.0', 8000)
+        >>> validaddr('127.0.0.1')
+        ('127.0.0.1', 8080)
+        >>> validaddr('127.0.0.1:8000')
+        ('127.0.0.1', 8000)
+        >>> validip('[::1]:80')
+        ('::1', 80)
+        >>> validaddr('fff')
+        Traceback (most recent call last):
+            ...
+        ValueError: fff is not a valid IP address/port
+    """
+    if "/" in string_:
+        return string_
+    else:
+        return validip(string_)
+
+
+def urlquote(val):
+    """
+    Quotes a string for use in a URL.
+
+        >>> urlquote('://?f=1&j=1')
+        '%3A//%3Ff%3D1%26j%3D1'
+        >>> urlquote(None)
+        ''
+        >>> urlquote(u'\u203d')
+        '%E2%80%BD'
+    """
+    if val is None:
+        return ""
+
+    val = str(val).encode("utf-8")
+    return quote(val)
+
+
+def httpdate(date_obj):
+    """
+    Formats a datetime object for use in HTTP headers.
+
+        >>> import datetime
+        >>> httpdate(datetime.datetime(1970, 1, 1, 1, 1, 1))
+        'Thu, 01 Jan 1970 01:01:01 GMT'
+    """
+    return date_obj.strftime("%a, %d %b %Y %H:%M:%S GMT")
+
+
+def parsehttpdate(string_):
+    """
+    Parses an HTTP date into a datetime object.
+
+        >>> parsehttpdate('Thu, 01 Jan 1970 01:01:01 GMT')
+        datetime.datetime(1970, 1, 1, 1, 1, 1)
+    """
+    try:
+        t = time.strptime(string_, "%a, %d %b %Y %H:%M:%S %Z")
+    except ValueError:
+        return None
+    return datetime.datetime(*t[:6])
+
+
+def htmlquote(text):
+    r"""
+    Encodes `text` for raw use in HTML.
+
+        >>> htmlquote(u"<'&\">")
+        u'&lt;&#39;&amp;&quot;&gt;'
+    """
+    text = text.replace("&", "&amp;")  # Must be done first!
+    text = text.replace("<", "&lt;")
+    text = text.replace(">", "&gt;")
+    text = text.replace("'", "&#39;")
+    text = text.replace('"', "&quot;")
+    return text
+
+
+def htmlunquote(text):
+    r"""
+    Decodes `text` that's HTML quoted.
+
+        >>> htmlunquote(u'&lt;&#39;&amp;&quot;&gt;')
+        u'<\'&">'
+    """
+    text = text.replace("&quot;", '"')
+    text = text.replace("&#39;", "'")
+    text = text.replace("&gt;", ">")
+    text = text.replace("&lt;", "<")
+    text = text.replace("&amp;", "&")  # Must be done last!
+    return text
+
+
+def websafe(val):
+    r"""
+    Converts `val` so that it is safe for use in Unicode HTML.
+
+        >>> websafe("<'&\">")
+        u'&lt;&#39;&amp;&quot;&gt;'
+        >>> websafe(None)
+        u''
+        >>> websafe(u'\u203d') == u'\u203d'
+        True
+    """
+    if val is None:
+        return ""
+
+    if isinstance(val, bytes):
+        val = val.decode("utf-8")
+    elif not isinstance(val, str):
+        val = str(val)
+
+    return htmlquote(val)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

web/py3helpers.py

@@ -0,0 +1,7 @@
+"""Utilities for make the code run both on Python2 and Python3.
+"""
+
+# Dictionary iteration
+iterkeys = lambda d: iter(d.keys())
+itervalues = lambda d: iter(d.values())
+iteritems = lambda d: iter(d.items())

web/session.py

@@ -0,0 +1,457 @@
+"""
+Session Management
+(from web.py)
+"""
+
+import datetime
+import os
+import os.path
+import shutil
+import threading
+import time
+from copy import deepcopy
+from hashlib import sha1
+
+from . import utils
+from . import webapi as web
+from .py3helpers import iteritems
+
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+from base64 import decodebytes, encodebytes
+
+__all__ = ["Session", "SessionExpired", "Store", "DiskStore", "DBStore", "MemoryStore"]
+
+web.config.session_parameters = utils.storage(
+    {
+        "cookie_name": "webpy_session_id",
+        "cookie_domain": None,
+        "cookie_path": None,
+        "samesite": None,
+        "timeout": 86400,  # 24 * 60 * 60, # 24 hours in seconds
+        "ignore_expiry": True,
+        "ignore_change_ip": True,
+        "secret_key": "fLjUfxqXtfNoIldA0A0J",
+        "expired_message": "Session expired",
+        "httponly": True,
+        "secure": False,
+    }
+)
+
+
+class SessionExpired(web.HTTPError):
+    def __init__(self, message):
+        web.HTTPError.__init__(self, "200 OK", {}, data=message)
+
+
+class Session:
+    """Session management for web.py"""
+
+    __slots__ = [
+        "store",
+        "_initializer",
+        "_last_cleanup_time",
+        "_config",
+        "_data",
+        "__getitem__",
+        "__setitem__",
+        "__delitem__",
+    ]
+
+    def __init__(self, app, store, initializer=None):
+        self.store = store
+        self._initializer = initializer
+        self._last_cleanup_time = 0
+        self._config = utils.storage(web.config.session_parameters)
+        self._data = utils.threadeddict()
+
+        self.__getitem__ = self._data.__getitem__
+        self.__setitem__ = self._data.__setitem__
+        self.__delitem__ = self._data.__delitem__
+
+        if app:
+            app.add_processor(self._processor)
+
+    def __contains__(self, name):
+        return name in self._data
+
+    def __getattr__(self, name):
+        return getattr(self._data, name)
+
+    def __setattr__(self, name, value):
+        if name in self.__slots__:
+            object.__setattr__(self, name, value)
+        else:
+            setattr(self._data, name, value)
+
+    def __delattr__(self, name):
+        delattr(self._data, name)
+
+    def _processor(self, handler):
+        """Application processor to setup session for every request"""
+
+        self._cleanup()
+        self._load()
+
+        try:
+            return handler()
+        finally:
+            self._save()
+
+    def _load(self):
+        """Load the session from the store, by the id from cookie"""
+        cookie_name = self._config.cookie_name
+        self.session_id = web.cookies().get(cookie_name)
+
+        # protection against session_id tampering
+        if self.session_id and not self._valid_session_id(self.session_id):
+            self.session_id = None
+
+        self._check_expiry()
+        if self.session_id:
+            d = self.store[self.session_id]
+            self.update(d)
+            self._validate_ip()
+
+        if not self.session_id:
+            self.session_id = self._generate_session_id()
+
+            if self._initializer:
+                if isinstance(self._initializer, dict):
+                    self.update(deepcopy(self._initializer))
+                elif hasattr(self._initializer, "__call__"):
+                    self._initializer()
+
+        self.ip = web.ctx.ip
+
+    def _check_expiry(self):
+        # check for expiry
+        if self.session_id and self.session_id not in self.store:
+            if self._config.ignore_expiry:
+                self.session_id = None
+            else:
+                return self.expired()
+
+    def _validate_ip(self):
+        # check for change of IP
+        if self.session_id and self.get("ip", None) != web.ctx.ip:
+            if not self._config.ignore_change_ip:
+                return self.expired()
+
+    def _save(self):
+        current_values = dict(self._data)
+        del current_values["session_id"]
+        del current_values["ip"]
+
+        if not self.get("_killed"):
+            self._setcookie(self.session_id)
+            self.store[self.session_id] = dict(self._data)
+        else:
+            if web.cookies().get(self._config.cookie_name):
+                self._setcookie(self.session_id, expires=-1)
+
+    def _setcookie(self, session_id, expires="", **kw):
+        cookie_name = self._config.cookie_name
+        cookie_domain = self._config.cookie_domain
+        cookie_path = self._config.cookie_path
+        httponly = self._config.httponly
+        secure = self._config.secure
+        samesite = kw.get("samesite", self._config.get("samesite", None))
+        web.setcookie(
+            cookie_name,
+            session_id,
+            expires=expires,
+            domain=cookie_domain,
+            httponly=httponly,
+            secure=secure,
+            path=cookie_path,
+            samesite=samesite,
+        )
+
+    def _generate_session_id(self):
+        """Generate a random id for session"""
+
+        while True:
+            rand = os.urandom(16)
+            now = time.time()
+            secret_key = self._config.secret_key
+
+            hashable = f"{rand}{now}{utils.safestr(web.ctx.ip)}{secret_key}"
+            session_id = sha1(hashable.encode("utf-8")).hexdigest()
+            if session_id not in self.store:
+                break
+        return session_id
+
+    def _valid_session_id(self, session_id):
+        rx = utils.re_compile("^[0-9a-fA-F]+$")
+        return rx.match(session_id)
+
+    def _cleanup(self):
+        """Cleanup the stored sessions"""
+        current_time = time.time()
+        timeout = self._config.timeout
+        if current_time - self._last_cleanup_time > timeout:
+            self.store.cleanup(timeout)
+            self._last_cleanup_time = current_time
+
+    def expired(self):
+        """Called when an expired session is atime"""
+        self._killed = True
+        self._save()
+        raise SessionExpired(self._config.expired_message)
+
+    def kill(self):
+        """Kill the session, make it no longer available"""
+        del self.store[self.session_id]
+        self._killed = True
+
+
+class Store:
+    """Base class for session stores"""
+
+    def __contains__(self, key):
+        raise NotImplementedError()
+
+    def __getitem__(self, key):
+        raise NotImplementedError()
+
+    def __setitem__(self, key, value):
+        raise NotImplementedError()
+
+    def cleanup(self, timeout):
+        """removes all the expired sessions"""
+        raise NotImplementedError()
+
+    def encode(self, session_dict):
+        """encodes session dict as a string"""
+        pickled = pickle.dumps(session_dict)
+        return encodebytes(pickled)
+
+    def decode(self, session_data):
+        """decodes the data to get back the session dict"""
+        if isinstance(session_data, str):
+            session_data = session_data.encode()
+
+        pickled = decodebytes(session_data)
+        return pickle.loads(pickled)
+
+
+class DiskStore(Store):
+    """
+    Store for saving a session on disk.
+
+        >>> import tempfile
+        >>> root = tempfile.mkdtemp()
+        >>> s = DiskStore(root)
+        >>> s['a'] = 'foo'
+        >>> s['a']
+        'foo'
+        >>> time.sleep(0.01)
+        >>> s.cleanup(0.01)
+        >>> s['a']
+        Traceback (most recent call last):
+            ...
+        KeyError: 'a'
+    """
+
+    def __init__(self, root):
+        # if the storage root doesn't exists, create it.
+        if not os.path.exists(root):
+            os.makedirs(os.path.abspath(root))
+        self.root = root
+
+    def _get_path(self, key):
+        if os.path.sep in key:
+            raise ValueError("Bad key: %s" % repr(key))
+        return os.path.join(self.root, key)
+
+    def __contains__(self, key):
+        path = self._get_path(key)
+        return os.path.exists(path)
+
+    def __getitem__(self, key):
+        path = self._get_path(key)
+
+        if os.path.exists(path):
+            with open(path, "rb") as fh:
+                pickled = fh.read()
+            return self.decode(pickled)
+        else:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        path = self._get_path(key)
+        pickled = self.encode(value)
+        try:
+            tname = path + "." + threading.current_thread().getName()
+            f = open(tname, "wb")
+            try:
+                f.write(pickled)
+            finally:
+                f.close()
+                shutil.move(tname, path)  # atomary operation
+        except OSError:
+            pass
+
+    def __delitem__(self, key):
+        path = self._get_path(key)
+        if os.path.exists(path):
+            os.remove(path)
+
+    def cleanup(self, timeout):
+        if not os.path.isdir(self.root):
+            return
+
+        now = time.time()
+        for f in os.listdir(self.root):
+            path = self._get_path(f)
+            atime = os.stat(path).st_atime
+            if now - atime > timeout:
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+                else:
+                    os.remove(path)
+
+
+class DBStore(Store):
+    """Store for saving a session in database
+    Needs a table with the following columns:
+
+        session_id CHAR(128) UNIQUE NOT NULL,
+        atime DATETIME NOT NULL default current_timestamp,
+        data TEXT
+    """
+
+    def __init__(self, db, table_name):
+        self.db = db
+        self.table = table_name
+
+    def __contains__(self, key):
+        data = self.db.select(self.table, where="session_id=$key", vars=locals())
+        return bool(list(data))
+
+    def __getitem__(self, key):
+        now = datetime.datetime.now()
+        try:
+            s = self.db.select(self.table, where="session_id=$key", vars=locals())[0]
+            self.db.update(
+                self.table, where="session_id=$key", atime=now, vars=locals()
+            )
+        except IndexError:
+            raise KeyError(key)
+        else:
+            return self.decode(s.data)
+
+    def __setitem__(self, key, value):
+        # Remove the leading `b` of bytes object (`b"..."`), otherwise encoded
+        # value is invalid base64 format.
+        pickled = self.encode(value).decode()
+
+        now = datetime.datetime.now()
+        if key in self:
+            self.db.update(
+                self.table,
+                where="session_id=$key",
+                data=pickled,
+                atime=now,
+                vars=locals(),
+            )
+        else:
+            self.db.insert(self.table, False, session_id=key, atime=now, data=pickled)
+
+    def __delitem__(self, key):
+        self.db.delete(self.table, where="session_id=$key", vars=locals())
+
+    def cleanup(self, timeout):
+        timeout = datetime.timedelta(
+            timeout / (24.0 * 60 * 60)
+        )  # timedelta takes numdays as arg
+        last_allowed_time = datetime.datetime.now() - timeout
+        self.db.delete(self.table, where="$last_allowed_time > atime", vars=locals())
+
+
+class ShelfStore:
+    """Store for saving session using `shelve` module.
+
+        import shelve
+        store = ShelfStore(shelve.open('session.shelf'))
+
+    XXX: is shelve thread-safe?
+    """
+
+    def __init__(self, shelf):
+        self.shelf = shelf
+
+    def __contains__(self, key):
+        return key in self.shelf
+
+    def __getitem__(self, key):
+        atime, v = self.shelf[key]
+        self[key] = v  # update atime
+        return v
+
+    def __setitem__(self, key, value):
+        self.shelf[key] = time.time(), value
+
+    def __delitem__(self, key):
+        try:
+            del self.shelf[key]
+        except KeyError:
+            pass
+
+    def cleanup(self, timeout):
+        now = time.time()
+        for k in self.shelf:
+            atime, v = self.shelf[k]
+            if now - atime > timeout:
+                del self[k]
+
+
+class MemoryStore(Store):
+    """Store for saving a session in memory.
+    Useful where there is limited fs writes on the disk, like
+    flash memories
+
+    Data will be saved into a dict:
+    k: (time, pydata)
+    """
+
+    def __init__(self, d_store=None):
+        if d_store is None:
+            d_store = {}
+        self.d_store = d_store
+
+    def __contains__(self, key):
+        return key in self.d_store
+
+    def __getitem__(self, key):
+        """Return the value and update the last seen value"""
+        t, value = self.d_store[key]
+        self.d_store[key] = (time.time(), value)
+        return value
+
+    def __setitem__(self, key, value):
+        self.d_store[key] = (time.time(), value)
+
+    def __delitem__(self, key):
+        del self.d_store[key]
+
+    def cleanup(self, timeout):
+        now = time.time()
+        to_del = []
+        for k, (atime, value) in iteritems(self.d_store):
+            if now - atime > timeout:
+                to_del.append(k)
+
+        # to avoid exception on "dict change during iterations"
+        for k in to_del:
+            del self.d_store[k]
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

web/test.py

@@ -0,0 +1,55 @@
+"""test utilities
+(part of web.py)
+"""
+import doctest
+import sys
+import unittest
+
+TestCase = unittest.TestCase
+TestSuite = unittest.TestSuite
+
+
+def load_modules(names):
+    return [__import__(name, None, None, "x") for name in names]
+
+
+def module_suite(module, classnames=None):
+    """Makes a suite from a module."""
+    if classnames:
+        return unittest.TestLoader().loadTestsFromNames(classnames, module)
+    elif hasattr(module, "suite"):
+        return module.suite()
+    else:
+        return unittest.TestLoader().loadTestsFromModule(module)
+
+
+def doctest_suite(module_names):
+    """Makes a test suite from doctests."""
+    suite = TestSuite()
+    for mod in load_modules(module_names):
+        suite.addTest(doctest.DocTestSuite(mod))
+    return suite
+
+
+def suite(module_names):
+    """Creates a suite from multiple modules."""
+    suite = TestSuite()
+    for mod in load_modules(module_names):
+        suite.addTest(module_suite(mod))
+    return suite
+
+
+def runTests(suite):
+    runner = unittest.TextTestRunner()
+    return runner.run(suite)
+
+
+def main(suite=None):
+    if not suite:
+        main_module = __import__("__main__")
+        # allow command line switches
+        args = [a for a in sys.argv[1:] if not a.startswith("-")]
+        suite = module_suite(main_module, args or None)
+
+    result = runTests(suite)
+    sys.exit(not result.wasSuccessful())

web/webapi.py

@@ -0,0 +1,667 @@
+"""
+Web API (wrapper around WSGI)
+(from web.py)
+"""
+
+import cgi
+import pprint
+import sys
+import tempfile
+from http.cookies import CookieError, Morsel, SimpleCookie
+from io import BytesIO
+from urllib.parse import quote, unquote, urljoin
+
+from .utils import dictadd, intget, safestr, storage, storify, threadeddict
+
+__all__ = [
+    "config",
+    "header",
+    "debug",
+    "input",
+    "data",
+    "setcookie",
+    "cookies",
+    "ctx",
+    "HTTPError",
+    # 200, 201, 202, 204
+    "OK",
+    "Created",
+    "Accepted",
+    "NoContent",
+    "ok",
+    "created",
+    "accepted",
+    "nocontent",
+    # 301, 302, 303, 304, 307
+    "Redirect",
+    "Found",
+    "SeeOther",
+    "NotModified",
+    "TempRedirect",
+    "redirect",
+    "found",
+    "seeother",
+    "notmodified",
+    "tempredirect",
+    # 400, 401, 403, 404, 405, 406, 409, 410, 412, 415, 451
+    "BadRequest",
+    "Unauthorized",
+    "Forbidden",
+    "NotFound",
+    "NoMethod",
+    "NotAcceptable",
+    "Conflict",
+    "Gone",
+    "PreconditionFailed",
+    "UnsupportedMediaType",
+    "UnavailableForLegalReasons",
+    "badrequest",
+    "unauthorized",
+    "forbidden",
+    "notfound",
+    "nomethod",
+    "notacceptable",
+    "conflict",
+    "gone",
+    "preconditionfailed",
+    "unsupportedmediatype",
+    "unavailableforlegalreasons",
+    # 500
+    "InternalError",
+    "internalerror",
+]
+
+config = storage()
+config.__doc__ = """
+A configuration object for various aspects of web.py.
+
+`debug`
+   : when True, enables reloading, disabled template caching and sets internalerror to debugerror.
+"""
+
+
+class HTTPError(Exception):
+    def __init__(self, status, headers={}, data=""):
+        ctx.status = status
+        for k, v in headers.items():
+            header(k, v)
+        self.data = data
+        Exception.__init__(self, status)
+
+
+def _status_code(status, data=None, classname=None, docstring=None):
+    if data is None:
+        data = status.split(" ", 1)[1]
+    classname = status.split(" ", 1)[1].replace(
+        " ", ""
+    )  # 304 Not Modified -> NotModified
+    docstring = docstring or "`%s` status" % status
+
+    def __init__(self, data=data, headers={}):
+        HTTPError.__init__(self, status, headers, data)
+
+    # trick to create class dynamically with dynamic docstring.
+    return type(
+        classname, (HTTPError, object), {"__doc__": docstring, "__init__": __init__}
+    )
+
+
+ok = OK = _status_code("200 OK", data="")
+created = Created = _status_code("201 Created")
+accepted = Accepted = _status_code("202 Accepted")
+nocontent = NoContent = _status_code("204 No Content")
+
+
+class Redirect(HTTPError):
+    """A `301 Moved Permanently` redirect."""
+
+    def __init__(self, url, status="301 Moved Permanently", absolute=False):
+        """
+        Returns a `status` redirect to the new URL.
+        `url` is joined with the base URL so that things like
+        `redirect("about") will work properly.
+        """
+        newloc = urljoin(ctx.path, url)
+
+        if newloc.startswith("/"):
+            if absolute:
+                home = ctx.realhome
+            else:
+                home = ctx.home
+            newloc = home + newloc
+
+        headers = {"Content-Type": "text/html", "Location": newloc}
+        HTTPError.__init__(self, status, headers, "")
+
+
+redirect = Redirect
+
+
+class Found(Redirect):
+    """A `302 Found` redirect."""
+
+    def __init__(self, url, absolute=False):
+        Redirect.__init__(self, url, "302 Found", absolute=absolute)
+
+
+found = Found
+
+
+class SeeOther(Redirect):
+    """A `303 See Other` redirect."""
+
+    def __init__(self, url, absolute=False):
+        Redirect.__init__(self, url, "303 See Other", absolute=absolute)
+
+
+seeother = SeeOther
+
+
+class NotModified(HTTPError):
+    """A `304 Not Modified` status."""
+
+    def __init__(self):
+        HTTPError.__init__(self, "304 Not Modified")
+
+
+notmodified = NotModified
+
+
+class TempRedirect(Redirect):
+    """A `307 Temporary Redirect` redirect."""
+
+    def __init__(self, url, absolute=False):
+        Redirect.__init__(self, url, "307 Temporary Redirect", absolute=absolute)
+
+
+tempredirect = TempRedirect
+
+
+class BadRequest(HTTPError):
+    """`400 Bad Request` error."""
+
+    message = "bad request"
+
+    def __init__(self, message=None):
+        status = "400 Bad Request"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+badrequest = BadRequest
+
+
+class Unauthorized(HTTPError):
+    """`401 Unauthorized` error."""
+
+    message = "unauthorized"
+
+    def __init__(self, message=None):
+        status = "401 Unauthorized"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+unauthorized = Unauthorized
+
+
+class Forbidden(HTTPError):
+    """`403 Forbidden` error."""
+
+    message = "forbidden"
+
+    def __init__(self, message=None):
+        status = "403 Forbidden"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+forbidden = Forbidden
+
+
+class _NotFound(HTTPError):
+    """`404 Not Found` error."""
+
+    message = "not found"
+
+    def __init__(self, message=None):
+        status = "404 Not Found"
+        headers = {"Content-Type": "text/html; charset=utf-8"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+def NotFound(message=None):
+    """Returns HTTPError with '404 Not Found' error from the active application."""
+    if message:
+        return _NotFound(message)
+    elif ctx.get("app_stack"):
+        return ctx.app_stack[-1].notfound()
+    else:
+        return _NotFound()
+
+
+notfound = NotFound
+
+
+class NoMethod(HTTPError):
+    """A `405 Method Not Allowed` error."""
+
+    message = "method not allowed"
+
+    def __init__(self, cls=None):
+        status = "405 Method Not Allowed"
+        headers = {}
+        headers["Content-Type"] = "text/html"
+
+        methods = ["GET", "HEAD", "POST", "PUT", "DELETE"]
+        if cls:
+            methods = [method for method in methods if hasattr(cls, method)]
+
+        headers["Allow"] = ", ".join(methods)
+        HTTPError.__init__(self, status, headers, self.message)
+
+
+nomethod = NoMethod
+
+
+class NotAcceptable(HTTPError):
+    """`406 Not Acceptable` error."""
+
+    message = "not acceptable"
+
+    def __init__(self, message=None):
+        status = "406 Not Acceptable"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+notacceptable = NotAcceptable
+
+
+class Conflict(HTTPError):
+    """`409 Conflict` error."""
+
+    message = "conflict"
+
+    def __init__(self, message=None):
+        status = "409 Conflict"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+conflict = Conflict
+
+
+class Gone(HTTPError):
+    """`410 Gone` error."""
+
+    message = "gone"
+
+    def __init__(self, message=None):
+        status = "410 Gone"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+gone = Gone
+
+
+class PreconditionFailed(HTTPError):
+    """`412 Precondition Failed` error."""
+
+    message = "precondition failed"
+
+    def __init__(self, message=None):
+        status = "412 Precondition Failed"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+preconditionfailed = PreconditionFailed
+
+
+class UnsupportedMediaType(HTTPError):
+    """`415 Unsupported Media Type` error."""
+
+    message = "unsupported media type"
+
+    def __init__(self, message=None):
+        status = "415 Unsupported Media Type"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+unsupportedmediatype = UnsupportedMediaType
+
+
+class _UnavailableForLegalReasons(HTTPError):
+    """`451 Unavailable For Legal Reasons` error."""
+
+    message = "unavailable for legal reasons"
+
+    def __init__(self, message=None):
+        status = "451 Unavailable For Legal Reasons"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+def UnavailableForLegalReasons(message=None):
+    """Returns HTTPError with '415 Unavailable For Legal Reasons' error from the active application."""
+    if message:
+        return _UnavailableForLegalReasons(message)
+    elif ctx.get("app_stack"):
+        return ctx.app_stack[-1].unavailableforlegalreasons()
+    else:
+        return _UnavailableForLegalReasons()
+
+
+unavailableforlegalreasons = UnavailableForLegalReasons
+
+
+class _InternalError(HTTPError):
+    """500 Internal Server Error`."""
+
+    message = "internal server error"
+
+    def __init__(self, message=None):
+        status = "500 Internal Server Error"
+        headers = {"Content-Type": "text/html"}
+        HTTPError.__init__(self, status, headers, message or self.message)
+
+
+def InternalError(message=None):
+    """Returns HTTPError with '500 internal error' error from the active application."""
+    if message:
+        return _InternalError(message)
+    elif ctx.get("app_stack"):
+        return ctx.app_stack[-1].internalerror()
+    else:
+        return _InternalError()
+
+
+internalerror = InternalError
+
+
+class cgiFieldStorage(cgi.FieldStorage):
+    """
+    Subclass cgi.FieldStorage, as read_binary expects fp to return
+    bytes. If the headers do not contain a content-disposition with a
+    filename, cgi.FieldStorage's make_file will create a TemporaryFile
+    with `w+` flags. The write to that temporary file will fail, due
+    to incorrect encoding in Python 3.
+    """
+
+    def make_file(self, binary=None):
+        """
+        For backwards compatibility with Python 2, make_file accepted
+        a binary flag. This was unused, and removed in Python 3.
+        """
+        return tempfile.TemporaryFile("wb+")
+
+
+def header(hdr, value, unique=False):
+    """
+    Adds the header `hdr: value` with the response.
+
+    If `unique` is True and a header with that name already exists,
+    it doesn't add a new one.
+    """
+    hdr, value = safestr(hdr), safestr(value)
+    # protection against HTTP response splitting attack
+    if "\n" in hdr or "\r" in hdr or "\n" in value or "\r" in value:
+        raise ValueError("invalid characters in header")
+    if unique is True:
+        for h, v in ctx.headers:
+            if h.lower() == hdr.lower():
+                return
+
+    ctx.headers.append((hdr, value))
+
+
+def rawinput(method=None):
+    """Returns storage object with GET or POST arguments."""
+    method = method or "both"
+
+    def dictify(fs):
+        # hack to make web.input work with enctype='text/plain.
+        if fs.list is None:
+            fs.list = []
+
+        return {k: fs[k] for k in fs}
+
+    e = ctx.env.copy()
+    a = b = {}
+
+    if method.lower() in ["both", "post", "put", "patch"]:
+        if e["REQUEST_METHOD"] in ["POST", "PUT", "PATCH"]:
+            if e.get("CONTENT_TYPE", "").lower().startswith("multipart/"):
+                # since wsgi.input is directly passed to cgi.FieldStorage,
+                # it can not be called multiple times. Saving the FieldStorage
+                # object in ctx to allow calling web.input multiple times.
+                a = ctx.get("_fieldstorage")
+                if not a:
+                    fp = e["wsgi.input"]
+                    a = cgiFieldStorage(fp=fp, environ=e, keep_blank_values=1)
+                    ctx._fieldstorage = a
+            else:
+                d = data()
+                if isinstance(d, str):
+                    d = d.encode("utf-8")
+                fp = BytesIO(d)
+                a = cgiFieldStorage(fp=fp, environ=e, keep_blank_values=1)
+            a = dictify(a)
+
+    if method.lower() in ["both", "get"]:
+        e["REQUEST_METHOD"] = "GET"
+        b = dictify(cgiFieldStorage(environ=e, keep_blank_values=1))
+
+    def process_fieldstorage(fs):
+        if isinstance(fs, list):
+            return [process_fieldstorage(x) for x in fs]
+        elif fs.filename is None:
+            return fs.value
+        else:
+            return fs
+
+    return storage([(k, process_fieldstorage(v)) for k, v in dictadd(b, a).items()])
+
+
+def input(*requireds, **defaults):
+    """
+    Returns a `storage` object with the GET and POST arguments.
+    See `storify` for how `requireds` and `defaults` work.
+    """
+    _method = defaults.pop("_method", "both")
+    out = rawinput(_method)
+    try:
+        defaults.setdefault("_unicode", True)  # force unicode conversion by default.
+        return storify(out, *requireds, **defaults)
+    except KeyError:
+        raise badrequest()
+
+
+def data():
+    """Returns the data sent with the request."""
+    if "data" not in ctx:
+        if ctx.env.get("HTTP_TRANSFER_ENCODING") == "chunked":
+            ctx.data = ctx.env["wsgi.input"].read()
+        else:
+            cl = intget(ctx.env.get("CONTENT_LENGTH"), 0)
+            ctx.data = ctx.env["wsgi.input"].read(cl)
+    return ctx.data
+
+
+def setcookie(
+    name,
+    value,
+    expires="",
+    domain=None,
+    secure=False,
+    httponly=False,
+    path=None,
+    samesite=None,
+):
+    """Sets a cookie."""
+    morsel = Morsel()
+    name, value = safestr(name), safestr(value)
+    morsel.set(name, value, quote(value))
+    if isinstance(expires, int) and expires < 0:
+        expires = -1000000000
+    morsel["expires"] = expires
+    morsel["path"] = path or ctx.homepath + "/"
+    if domain:
+        morsel["domain"] = domain
+    if secure:
+        morsel["secure"] = secure
+    if httponly:
+        morsel["httponly"] = True
+    value = morsel.OutputString()
+    if samesite and samesite.lower() in ("strict", "lax", "none"):
+        value += "; SameSite=%s" % samesite
+    header("Set-Cookie", value)
+
+
+def parse_cookies(http_cookie):
+    r"""Parse a HTTP_COOKIE header and return dict of cookie names and decoded values.
+
+    >>> sorted(parse_cookies('').items())
+    []
+    >>> sorted(parse_cookies('a=1').items())
+    [('a', '1')]
+    >>> sorted(parse_cookies('a=1%202').items())
+    [('a', '1 2')]
+    >>> sorted(parse_cookies('a=Z%C3%A9Z').items())
+    [('a', 'Z\xc3\xa9Z')]
+    >>> sorted(parse_cookies('a=1; b=2; c=3').items())
+    [('a', '1'), ('b', '2'), ('c', '3')]
+
+    # TODO: cclauss re-enable this test
+    # >>> sorted(parse_cookies('a=1; b=w("x")|y=z; c=3').items())
+    # [('a', '1'), ('b', 'w('), ('c', '3')]
+
+    >>> sorted(parse_cookies('a=1; b=w(%22x%22)|y=z; c=3').items())
+    [('a', '1'), ('b', 'w("x")|y=z'), ('c', '3')]
+
+    >>> sorted(parse_cookies('keebler=E=mc2').items())
+    [('keebler', 'E=mc2')]
+    >>> sorted(parse_cookies(r'keebler="E=mc2; L=\"Loves\"; fudge=\012;"').items())
+    [('keebler', 'E=mc2; L="Loves"; fudge=\n;')]
+    """
+    # print "parse_cookies"
+    if '"' in http_cookie:
+        # HTTP_COOKIE has quotes in it, use slow but correct cookie parsing
+        cookie = SimpleCookie()
+        try:
+            cookie.load(http_cookie)
+        except CookieError:
+            # If HTTP_COOKIE header is malformed, try at least to load the cookies we can by
+            # first splitting on ';' and loading each attr=value pair separately
+            cookie = SimpleCookie()
+            for attr_value in http_cookie.split(";"):
+                try:
+                    cookie.load(attr_value)
+                except CookieError:
+                    pass
+        cookies = {k: unquote(v.value) for k, v in cookie.items()}
+    else:
+        # HTTP_COOKIE doesn't have quotes, use fast cookie parsing
+        cookies = {}
+        for key_value in http_cookie.split(";"):
+            key_value = key_value.split("=", 1)
+            if len(key_value) == 2:
+                key, value = key_value
+                cookies[key.strip()] = unquote(value.strip())
+    return cookies
+
+
+def cookies(*requireds, **defaults):
+    """Returns a `storage` object with all the request cookies in it.
+
+    See `storify` for how `requireds` and `defaults` work.
+
+    This is forgiving on bad HTTP_COOKIE input, it tries to parse at least
+    the cookies it can.
+
+    The values are converted to unicode if _unicode=True is passed.
+    """
+    # parse cookie string and cache the result for next time.
+    if "_parsed_cookies" not in ctx:
+        http_cookie = ctx.env.get("HTTP_COOKIE", "")
+        ctx._parsed_cookies = parse_cookies(http_cookie)
+
+    try:
+        return storify(ctx._parsed_cookies, *requireds, **defaults)
+    except KeyError:
+        badrequest()
+        raise StopIteration()
+
+
+def debug(*args):
+    """
+    Prints a prettyprinted version of `args` to stderr.
+    """
+    try:
+        out = ctx.environ["wsgi.errors"]
+    except:
+        out = sys.stderr
+    for arg in args:
+        print(pprint.pformat(arg), file=out)
+    return ""
+
+
+def _debugwrite(x):
+    try:
+        out = ctx.environ["wsgi.errors"]
+    except:
+        out = sys.stderr
+    out.write(x)
+
+
+debug.write = _debugwrite
+
+ctx = context = threadeddict()
+
+ctx.__doc__ = """
+A `storage` object containing various information about the request:
+
+`environ` (aka `env`)
+   : A dictionary containing the standard WSGI environment variables.
+
+`host`
+   : The domain (`Host` header) requested by the user.
+
+`home`
+   : The base path for the application.
+
+`ip`
+   : The IP address of the requester.
+
+`method`
+   : The HTTP method used.
+
+`path`
+   : The path request.
+
+`query`
+   : If there are no query arguments, the empty string. Otherwise, a `?` followed
+     by the query string.
+
+`fullpath`
+   : The full path requested, including query arguments (`== path + query`).
+
+### Response Data
+
+`status` (default: "200 OK")
+   : The status code to be used in the response.
+
+`headers`
+   : A list of 2-tuples to be used in the response.
+
+`output`
+   : A string to be used as the response.
+"""
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

web/wsgi.py

@@ -0,0 +1,87 @@
+"""
+WSGI Utilities
+(from web.py)
+"""
+
+import os
+import sys
+
+from . import httpserver
+from . import webapi as web
+from .net import validaddr
+from .utils import intget, listget
+
+
+def runfcgi(func, addr=("localhost", 8000)):
+    """Runs a WSGI function as a FastCGI server."""
+    import flup.server.fcgi as flups
+
+    return flups.WSGIServer(func, multiplexed=True, bindAddress=addr, debug=False).run()
+
+
+def runscgi(func, addr=("localhost", 4000)):
+    """Runs a WSGI function as an SCGI server."""
+    import flup.server.scgi as flups
+
+    return flups.WSGIServer(func, bindAddress=addr, debug=False).run()
+
+
+def runwsgi(func):
+    """
+    Runs a WSGI-compatible `func` using FCGI, SCGI, or a simple web server,
+    as appropriate based on context and `sys.argv`.
+    """
+
+    if "SERVER_SOFTWARE" in os.environ:  # cgi
+        os.environ["FCGI_FORCE_CGI"] = "Y"
+
+    # PHP_FCGI_CHILDREN is used by lighttpd fastcgi
+    if "PHP_FCGI_CHILDREN" in os.environ or "SERVER_SOFTWARE" in os.environ:
+        return runfcgi(func, None)
+
+    if "fcgi" in sys.argv or "fastcgi" in sys.argv:
+        args = sys.argv[1:]
+        if "fastcgi" in args:
+            args.remove("fastcgi")
+        elif "fcgi" in args:
+            args.remove("fcgi")
+
+        if args:
+            return runfcgi(func, validaddr(args[0]))
+        else:
+            return runfcgi(func, None)
+
+    if "scgi" in sys.argv:
+        args = sys.argv[1:]
+        args.remove("scgi")
+        if args:
+            return runscgi(func, validaddr(args[0]))
+        else:
+            return runscgi(func)
+
+    server_addr = validaddr(listget(sys.argv, 1, ""))
+    if "PORT" in os.environ:  # e.g. Heroku
+        server_addr = ("0.0.0.0", intget(os.environ["PORT"]))
+
+    return httpserver.runsimple(func, server_addr)
+
+
+def _is_dev_mode():
+    # Some embedded python interpreters won't have sys.arv
+    # For details, see https://github.com/webpy/webpy/issues/87
+    argv = getattr(sys, "argv", [])
+
+    # quick hack to check if the program is running in dev mode.
+    if (
+        "SERVER_SOFTWARE" in os.environ
+        or "PHP_FCGI_CHILDREN" in os.environ
+        or "fcgi" in argv
+        or "fastcgi" in argv
+        or "mod_wsgi" in argv
+    ):
+        return False
+    return True
+
+
+# When running the builtin-server, enable debug mode if not already set.
+web.config.setdefault("debug", _is_dev_mode())