balki/mail4one
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 5 Wiki Activity

Compare commits

base: balki:main
Branches Tags
balki:main
balki:starttls_header
balki:pylint
balki:webform
balki:py38
balki:py310
balki:v1.1
balki:v1.0
balki:v1.0-rc5
balki:v1.0-rc4
balki:v1.0-rc3
balki:v1.0-rc2
balki:1.0-rc1
..
compare: balki:1.0-rc1
Branches Tags
balki:starttls_header
balki:main
balki:pylint
balki:webform
balki:py38
balki:py310
balki:v1.1
balki:v1.0
balki:v1.0-rc5
balki:v1.0-rc4
balki:v1.0-rc3
balki:v1.0-rc2
balki:1.0-rc1

No commits in common. "main" and "1.0-rc1" have entirely different histories.
main ... 1.0-rc1

27 changed files with 343 additions and 788 deletions
Show Stats Expand all files Collapse all files

3
.gitignore vendored
View File

@ -3,5 +3,4 @@
__pycache__ __pycache__
dummy.py dummy.py
build build
mail4one*.pyz mail4one.pyz
deploy_configs/config.json

38
DEVNOTES.md
View File

@ -1,38 +0,0 @@
Notes for developers
## Running just one test
```
python -m unittest tests.test_pop.TestPop3.test_CAPA
```
## Patch for enable logging in test
Patch generated using below
```
git diff --patch -U1 tests >> ./DEVNOTES.md
```
Apply with below
```bash
git apply - <<PATCH
diff --git a/tests/test_pop.py b/tests/test_pop.py
index 55c1a91..a825665 100644
--- a/tests/test_pop.py
+++ b/tests/test_pop.py
@@ -55,3 +55,3 @@ def setUpModule() -> None:
global MAILS_PATH
- logging.basicConfig(level=logging.CRITICAL)
+ logging.basicConfig(level=logging.DEBUG)
td = tempfile.TemporaryDirectory(prefix="m41.pop.")
PATCH
```
## pylint
```
pylint mail4one/*py > /tmp/errs
vim +"cfile /tmp/errs"
```

79
Makefile
View File

@ -1,11 +1,24 @@
# Needs python3 >= 3.9, sed, git for build
mail4one.pyz: requirements.txt mail4one/*py
python3 -m pip install -r requirements.txt --no-compile --target build shell:
MYPYPATH=`pipenv --venv`/lib/python3.11/site-packages pipenv shell
test:
pipenv run python -m unittest discover
docker-tests:
docker run --pull=always -v `pwd`:/app -w /app --rm -it python:3.11-alpine sh runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm -it python:3.10-alpine sh runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm -it python:3.11 sh runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm -it python:3.10 sh runtests.sh
requirements.txt: Pipfile.lock
pipenv requirements > requirements.txt
build: clean requirements.txt
python3 -m pip install -r requirements.txt --target build
cp -r mail4one/ build/ cp -r mail4one/ build/
sed -i "s/DEVELOMENT/$(shell scripts/get_version.sh)/" build/mail4one/version.py python3 -m compileall build/mail4one -f
find build -name "*.pyi" -o -name "py.typed" | xargs -I typefile rm typefile
rm -rf build/bin build/aiosmtpd/{docs,tests,qa}
rm -rf build/mail4one/__pycache__
rm -rf build/*.dist-info rm -rf build/*.dist-info
python3 -m zipapp \ python3 -m zipapp \
--output mail4one.pyz \ --output mail4one.pyz \
@ -13,58 +26,6 @@ mail4one.pyz: requirements.txt mail4one/*py
--main mail4one.server:main \ --main mail4one.server:main \
--compress build --compress build
.PHONY: build
build: clean mail4one.pyz
.PHONY: test
test: mail4one.pyz
PYTHONPATH=mail4one.pyz python3 -m unittest discover
.PHONY: clean
clean: clean:
rm -rf build rm -rf build
rm -rf mail4one.pyz rm -rf mail4one.pyz
.PHONY: docker-tests
docker-tests:
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.11-alpine sh scripts/runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.10-alpine sh scripts/runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.12 sh scripts/runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.11 sh scripts/runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.10 sh scripts/runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.9 sh scripts/runtests.sh
# ============================================================================
# Below targets for devs. Need pipenv, black installed
requirements.txt: Pipfile.lock
pipenv requirements > requirements.txt
.PHONY: format
format:
black mail4one/*py tests/*py
.PHONY: build-dev
build-dev: requirements.txt build
.PHONY: setup
setup:
pipenv install
.PHONY: cleanup
cleanup:
pipenv --rm
.PHONY: update
update:
rm requirements.txt Pipfile.lock
pipenv update
pipenv requirements > requirements.txt
.PHONY: shell
shell:
MYPYPATH=$(shell ls -d `pipenv --venv`/lib/python3*/site-packages) pipenv shell
.PHONY: dev-test
dev-test:
pipenv run python -m unittest discover

20
Pipfile.lock generated
View File

@ -18,28 +18,27 @@
"default": { "default": {
"aiosmtpd": { "aiosmtpd": {
"hashes": [ "hashes": [
"sha256:78d7b14f859ad0e6de252b47f9cf1ca6f1c82a8b0f10a9e39bec7e915a6aa5fe", "sha256:f821fe424b703b2ea391dc2df11d89d2afd728af27393e13cf1a3530f19fdc5e",
"sha256:a196922f1903e54c4d37c53415b7613056d39e2b1e8249f324b9ee7a439be0f1" "sha256:f9243b7dfe00aaf567da8728d891752426b51392174a34d2cf5c18053b63dcbc"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.8'", "version": "==1.4.4.post2"
"version": "==1.4.5"
}, },
"atpublic": { "atpublic": {
"hashes": [ "hashes": [
"sha256:d1c8cd931af7461f6d18bc6063383e8654d9e9ef19d58ee6dc01e8515bbf55df", "sha256:0f40433219e124edf115c6c363808ca6f0e1cfa7d160d86b2fb94793086d1294",
"sha256:df90de1162b1a941ee486f484691dc7c33123ee638ea5d6ca604061306e0fdde" "sha256:80057c55641253b86dcb68b524f82328172371b6547d4c7462a9127fbfbbabfc"
], ],
"markers": "python_version >= '3.8'", "markers": "python_version >= '3.8'",
"version": "==4.1.0" "version": "==4.0"
}, },
"attrs": { "attrs": {
"hashes": [ "hashes": [
"sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
"sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==23.2.0" "version": "==23.1.0"
}, },
"python-jata": { "python-jata": {
"hashes": [ "hashes": [
@ -47,7 +46,6 @@
"sha256:ff4cd7ca75c9a8306b69ef6e878c296a5602f3279c6f9a82b6105b8eba764760" "sha256:ff4cd7ca75c9a8306b69ef6e878c296a5602f3279c6f9a82b6105b8eba764760"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==1.2" "version": "==1.2"
} }
}, },

88
README.md
View File

@ -1,70 +1,44 @@
# Mail4one # mail4one
Personal mail server for a single user or a small family. Written in pure python with [minimal dependencies](Pipfile). Mail server for single user #asyncio #python
Designed for dynamic alias based workflow where a different alias is used for each purpose.
# Getting started ## Features
1. Get a domain name * smtp server with STARTTLS
1. Get a VPS (or a home server). Setup firewall rules for receive on port 25, 995, 465 * pop3 server with TLS
1. Setup [MX record](#dns-records-receiving) * Both running on single thread using asyncio
1. [Build](#building-from-source) / Download latest release - `mail4one.pyz` * Saves mails in simple Maildir format (i.e one file per email message)
1. Generate `config.json` from [config.sample](deploy_configs/config.sample) * After opening port, drops root privileges. So the process will not running as `nobody`
1. Run `./mail4one.pyz -c config.json`
1. Setup systemd service and TLS certificates. See [deploy_configs](deploy_configs/) for examples
# Sending email ## How to use
Mail4one only takes care of receiving and serving email. For sending email, use an external service like below echo -n "balki is awesome+<YOUR PASSWORD>" | sha256sum
pipenv install
sudo $(pipenv --venv)/bin/python ./run.py --certfile /etc/letsencrypt/live/your.domain.com/fullchain.pem --keyfile /etc/letsencrypt/live/your.domain.com/privkey.pem /var/mails --password_hash <PASSWORD_HASH_FROM_ABOVE>
* https://www.smtp2go.com/pricing/ ## Just pop server for debugging
* https://www.mailgun.com/pricing/
* https://sendgrid.com/free/
Most of them have generous free tier which is more than enough for personal use. pipenv run python -m mail4one.pop3 /path/to/mails 9995 your_password
Sending email is tricky. Even if everything is correctly setup (DMARC, DKIM, SPF), popular email vendors like google, microsoft may mark emails sent from your IP as spam for no reason. Hence using a dedicated service is the only reliable way to send emails. ## Nextups
# Community * Support sending emails - Also support for popular services like mailgun/sendgrid
* Smart assistant like functionality. For e.g.
* You don't need all emails of package deliver status. Just the latest one would be enough.
* Some type of emails can auto expire. Old newsletters are not very helpful
* Aggregate emails for weekend reading.
* Small webserver
* SPAM filtering - not that important as you can use unique addresses for each service. e.g. facebook@mydomian.com, bankac@mydomain.com, reddit@mydomain.com etc. You can easily figure out who sold your address to spammers and block it.
Original source is at https://gitea.balki.me/balki/mail4one ## Goals
* Intended to be used for one person. So won't have features that don't make sense in this context. e.g. LDAP AUTH, Mail quota, etc,
* Supports only python3.7. No plans to support older versions
For issues, pull requests, discussions, please use github mirror: https://github.com/mail4one/mail4one ## Known to work
* Server: Google Cloud f1-micro with Ubuntu 18.04 - Always Free instance
* Clients: thunderbird, evolution, k9mail
* smtp: Received email from all. Didn't see any drops. Tested from gmail, protonmail, reddit and few others
# Documentation ## Contribution
See files under [deploy_configs](deploy_configs/) for configuring and deploying to a standard systemd based linux system (e.g. debian, ubuntu, fedora, archlinux etc). [config.sample](deploy_configs/config.sample) has inline comments for more details. Feel free create github issue/discussions for support. Pull requests and issues welcome
## DNS Records (Receiving)
If you want to receive email for `john@example.com` and your VPS IP address is `1.2.3.4`. Following record needs to be created
|Type | Name | Target | Notes |
|------|------------------|----------------------|------------------------------------------------------|
| A | mail.example.com | `1.2.3.4` | |
| AAAA | mail.example.com | `abcd:1234::1234::1` | Optional, add if available |
| MX | example.com | `mail.example.com` | |
| MX | sub.example.com | `mail.example.com` | Optional, to receive emails like foo@sub.example.com |
For sending emails `DMARC`, `DKIM` and `SPF` records need to be set. Please refer to email [sending](#sending-email) provider for details.
# Building from source
Make sure to have make, git, python >= 3.9, and pip installed in your system and run below
make build
This should generate `mail4one.pyz` in current folder. This is a [executable python archive](https://docs.python.org/3/library/zipapp.html). Should be runnable as `./mail4one.pyz` or as `python3 mail4one.pyz`.
# Roadmap (Planned features for future)
* Other ways to install and update (PIP, AUR, docker etc)
* Write dedicated documentation
* Test with more email clients ([Thunderbird](https://www.thunderbird.net/) and [k9mail](https://k9mail.app/) are tested now)
* IMAP support
* Web UI for editing config ([WIP](https://github.com/mail4one/mail4one/tree/webform))
* Support email submission from client to forward to other senders or direct delivery
* Optional SPAM filtering
* Optional DMARC,SPF,DKIM verification
* Webmail Client
* Web UI to view graphs and smart reports

33
TODO.md Normal file
View File

@ -0,0 +1,33 @@
# TODO
DEV
1. unittests
1. Web interface
1. Custom email processing
1. Refactor smtp controller
1. pip installable package
1. Improve logging with timestamp, peer info, stats
1. custom smtp with auth
1. Filter domain name
2. Support multiple subdomains and save in different folders
1. smtp2go integration
1. metrics
2. imap
3. handle connection disconnect
4. rethink deleted items
MASTER
1. tests!
2. docker support
3. multi domain
4. [pop] auto delete
5. [pop] TOP command
6. merge with cloud_test branch
7. User timeout for POP
8. unittests
9. Web interface
10. Custom email processing
11. Refactor smtp controller
12. pip installable package
13. Listen on port 465 for smtp too

115
deploy_configs/README.md
View File

@ -1,115 +0,0 @@
# Deployment command line example
Example terminal session for deploying. ssh to your VPS and follow along. Minor differences may be required. e.g. if you are already root, skip `sudo`. If curl is missing, use wget.
## Check python version
Python version should be a supported (as of now 3.9 and above)
```sh
python3 -V
```
## Choose release
```sh
RELEASE=v1.0
```
## Download App
```sh
curl -OL "https://gitea.balki.me/balki/mail4one/releases/download/$RELEASE/mail4one.pyz"
chmod 555 mail4one.pyz
```
## Download sample configurations
```sh
curl -OL "https://gitea.balki.me/balki/mail4one/raw/tag/$RELEASE/deploy_configs/mail4one.service"
curl -OL "https://gitea.balki.me/balki/mail4one/raw/tag/$RELEASE/deploy_configs/mail4one.conf"
curl -OL "https://gitea.balki.me/balki/mail4one/raw/tag/$RELEASE/deploy_configs/mail4one_cert_copy.sh"
```
## Generate Password hash
This can be done in any machine. Do this once for each user. Every time a new hash is generated as a random salt is used. Even if you are using the same password for multiple clients, it is recommended to generate different hashes for each.
```sh
./mail4one.pyz -g
./mail4one.pyz -g <password> # also works but the password is saved in the shell commandline history
```
## Generate config.json
Edit [config.sample](config.sample) in your local machine and convert to config.json (See [here](./config.sample#L5) for some tools).
Then copy the config.json to your vps
```sh
scp config.json user@vps:~/
# or run below in vps terminal
cat > config.json
<paste json config from clibboard
<Ctrl + D>
# move to /etc
# This should show number of lines in your config
wc -l config.json
sudo mv config.json /etc/mail4one/config.json
```
## Create mail4one user
```sh
sudo mkdir -p /etc/sysusers.d/
sudo cp mail4one.conf /etc/sysusers.d/
sudo systemctl restart systemd-sysusers
# This should show the new user created
id mail4one
```
## Copy app
```sh
sudo cp mail4one.pyz /usr/local/bin/mail4one
# This should show executable permissions and should be owned by root
ls -l /usr/local/bin/mail4one
```
## Setup mail4one.service
```sh
sudo cp mail4one.service /etc/systemd/system/mail4one.service
sudo systemctl daemon-reload
sudo systemctl enable --now mail4one.service
systemctl status mail4one
```
Above command should fail as the TLS certificates don't exist yet.
## Setup TLS certificates
Install [certbot](https://certbot.eff.org/) and run below command. Follow instructions to create TLS certificates. Usually you want certificate for domain name like `mail.mydomain.com`
```sh
sudo certbot certonly
# **Edit** mail4one_cert_copy.sh to update your domain name
sudo cp mail4one_cert_copy.sh /etc/letsencrypt/renewal-hooks/deploy/
sudo chmod +x /etc/letsencrypt/renewal-hooks/deploy/mail4one_cert_copy.sh
# This will create and copy the certificates to the right path with correct permissions and ownership
sudo certbot certonly -d mail.mydomain.com --run-deploy-hooks --dry-run
```
## Restart service and check logs
```sh
sudo systemctl restart mail4one.service
systemctl status mail4one.service
cat /var/log/mail4one/mail4one.log
```
## Testing dns and firewall
In vps
```sh
mkdir test_dir
touch test_dir/{a,b,c}
cd test_dir
python3 -m http.server 25
```
In local machine or a browser
You should see file listing a, b, c. Repeat for port 465, 995 to make sure firewall rules and dns is working
```sh
curl http://mail.mydomain.com:25
```
If not working, refer to VPS settings and OS firewall settings.

144
deploy_configs/config.sample
View File

@ -1,144 +0,0 @@
# NOTE: Sample config is provided in yaml format for easy editing
# mail4one needs a json config, Please convert the config to json before passing to app
# This is to avoid yaml dependency in the app
#
# Some tools to convert to json:
# If you have `go` in your system (https://go.dev/)
# go run github.com/mikefarah/yq/v4@latest -oj -P . config.sample > config.json
#
# If you have `pipx` in your system (https://pypa.github.io/pipx/)
# pipx run yq . config.sample > config.json
#
# or a browser:
# https://onlineyamltools.com/convert-yaml-to-json
default_tls: # Will be used by both pop and smtp servers
# If using certbot(https://certbot.eff.org/),
# the following files will be here /etc/letsencrypt/live/<domain name>
# Use mail4one_cert_copy.sh to automaticallly copy on renewal
certfile: /var/lib/mail4one/certs/fullchain.pem
keyfile: /var/lib/mail4one/certs/privkey.pem
# default_host: '0.0.0.0'
logging:
# Setup logrotate(https://github.com/logrotate/logrotate) if needed
logfile: /var/log/mail4one/mail4one.log
mails_path: /var/lib/mail4one/mails
matches:
# only <to> address is matched. (sent by smtp RCPT command)
# address is converted to lowercase before matching
- name: example.com
addr_rexs:
- .*@example.com
- name: promotion-spammers
addrs:
- twitter@example.com
- random-app-not-used-anymore@example.com
- flyer-walmart@example.com
- name: john
addrs:
- john.doe@example.com # Tip: Dont use this. Always use a different alias, this way there is no address for spammers
- secret.name@example.com
- john.facebook@example.com
- name: jane
addrs:
- jane.doe@example.com
- jane.instagram@example.com
- name: jane-all
addr_rexs:
- jane.*@example.com
- name: shared
addrs:
- kidschool@example.com
- mortgage@example.com
- water@example.com
- electricity@example.com
- airbnb@example.com
boxes:
# Mails saved in maildir format under <mails_path>/<name>/new
- name: default_null_mbox # Means, the mails are dropped
rules:
- match_name: example.com
negate: true # Anything mail that does not match '.*@example.com'
stop_check: true # No further rules will be checked, mail is dropped
# Mailbox to store non-interesting emails but occasionally have a useful mail
# Create a second account in your email client and disable notification
- name: promotion-box
rules:
- match_name: promotion-spammers
stop_check: true
- name: johnsmbox
rules:
- match_name: john
- match_name: shared
## To receive all emails excluding jane's personal emails
# - match_name: jane
# negate: true
- name: janesmbox
rules:
- match_name: jane
- match_name: shared
- match_name: jane-all
- name: all
rules:
# matches all emails except those are not for 'example.com', which are dropped before
- match_name: default_match_all
users: # Used only by the pop server, smtp is for receiving mails only. No auth is used
- username: johnmobile
# Generated using below command. Will produce different hash each time (salt is random)
# ./mail4one.pyz -g johnsecretpassword
password_hash: AEH6JG3IZR3ASA2ORJHQ62YTR6PHFRP6PAXQ6RI2VZFXAT5M6VAATE373PGCUHBJTLIDOQV6UJKICP2JTKDE3QXP7ROJ227QYFQDAXPP4LY4TLPTEHUZG7D7X6VKWZ4BVCASYCD3SSNQ555AZPIFMDAV
mbox: johnsmbox
# **NOTE** Use different username for each email client.
# Otherwise emails may appear in only one client
- username: johnlaptop
password_hash: AEH6JG3IZR3ASA2ORJHQ62YTR6PHFRP6PAXQ6RI2VZFXAT5M6VAATE373PGCUHBJTLIDOQV6UJKICP2JTKDE3QXP7ROJ227QYFQDAXPP4LY4TLPTEHUZG7D7X6VKWZ4BVCASYCD3SSNQ555AZPIFMDAV
mbox: johnsmbox
# Second account to not clutter main mailbox. Disable notification for this account
- username: john-mobile-promotion
password_hash: AGBD47ZYBA7BMUQY25YDTYQWVPJFDBTLIICKFP2IL2GI4M7AO2LIVIZXTY6N25KBRLOEC7TLXGAFW7SSQEBKCG7U3FJNKW6RZWZBS3ABSP2U53BBIOCXZNWPXJGWAQ6WFXIF7T4YQJZD5QLF2OO4JZ45
mbox: promotion-box
- username: janemobile
password_hash: AGQNPATXU7PP7LDD6DZ4HFLUUHRJDHFQKKKRLVLGOIIEHC7TPOZF7NTXGDAIGDNHF62RAH4N44DB46O3VC4TBOLE5XHY6S77YPLTWCNAHGONEOZYO6YWJ3NHLKOHFJLF6BOHNMCI3RCPWXWXQPHSFDQR
mbox: janesmbox
- username: family_desktop # Catch all for backup
password_hash: AGBD47ZYBA7BMUQY25YDTYQWVPJFDBTLIICKFP2IL2GI4M7AO2LIVIZXTY6N25KBRLOEC7TLXGAFW7SSQEBKCG7U3FJNKW6RZWZBS3ABSP2U53BBIOCXZNWPXJGWAQ6WFXIF7T4YQJZD5QLF2OO4JZ45
mbox: all
servers:
- server_type: pop
## default values
# port: 995
# host: '0.0.0.0'
# tls: default # Uses default_tls config
- server_type: smtp
## default values
# port: 465
# host: '0.0.0.0'
# tls: default # Uses default_tls config
# tls: disable # disable tls and receive emails in plain text only
- server_type: smtp_starttls
## default values
# port: 25
# host: '0.0.0.0'
# tls: default # Uses default_tls config
# vim: ft=yaml

4
deploy_configs/mail4one.conf
View File

@ -1,7 +1,3 @@
# This file should be copied to /etc/sysusers.d/mail4one.conf
# Then either restart the system or run `systemctl restart systemd-sysusers`
# That should create a system user 'mail4one'
#
# See sysusers.d(5) for details. # See sysusers.d(5) for details.
u mail4one - "Personal Mail server" u mail4one - "Personal Mail server"

20
deploy_configs/mail4one.service
View File

@ -1,7 +1,4 @@
# This file should be copied to /etc/systemd/system/mail4one.service # mail4one.service
# Quickstart
# systemctl daemon-reload
# systemctl enable --now mail4one.service
[Unit] [Unit]
Description=Personal Mail server Description=Personal Mail server
@ -10,25 +7,16 @@ After=network.target network-online.target
Requires=network-online.target Requires=network-online.target
[Service] [Service]
# This user should already exist. See mail4one.conf for creating user with sysusers
User=mail4one User=mail4one
ExecStart=/usr/local/bin/mail4one --config /etc/mail4one/config.json ExecStart=/usr/local/bin/mail4one --config /etc/mail4one/config.json
PrivateTmp=true
# Below allows to bind to port < 1024. Standard ports are 25, 465, 995 ProtectSystem=full
AmbientCapabilities=CAP_NET_BIND_SERVICE AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
StateDirectory=mail4one/certs mail4one/mails StateDirectory=mail4one
StateDirectoryMode=0750
LogsDirectory=mail4one LogsDirectory=mail4one
WorkingDirectory=/var/lib/mail4one WorkingDirectory=/var/lib/mail4one
ProtectSystem=strict
PrivateTmp=true
ProtectHome=yes ProtectHome=yes
ProtectProc=invisible
NoNewPrivileges=yes
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

25
deploy_configs/mail4one_cert_copy.sh
View File

@ -1,27 +1,18 @@
#!/bin/sh #!/bin/sh
# certbot deploy hook to copy certificates to mail4one when renewed. # certbot deploy hook to copy certificates to mail4one when renewed.
# Initial setup, Install certbot(https://certbot.eff.org/) and run `certbot certonly` as root
# Doc: https://eff-certbot.readthedocs.io/en/latest/using.html#renewing-certificates
#
# This file is supposed to be copied to /etc/letsencrypt/renewal-hooks/deploy/ # This file is supposed to be copied to /etc/letsencrypt/renewal-hooks/deploy/
# Change the mail domain to the one on MX record # Change the mail domain to the one on MX record
set -eu set -x
if [ "$RENEWED_DOMAINS" = "mail.mydomain.com" ] if [ "$RENEWED_DOMAINS" = "mail.mydomain.com" ]
then then
app=mail4one mkdir -p /var/lib/mail4one/certs
appuser=$app chmod 500 /var/lib/mail4one/certs
certpath="/var/lib/$app/certs" chown mail4one:mail4one /var/lib/mail4one/certs
cp "$RENEWED_LINEAGE/fullchain.pem" /var/lib/mail4one/certs/
mkdir -p "$certpath" cp "$RENEWED_LINEAGE/privkey.pem" /var/lib/mail4one/certs/
chmod 750 "$certpath" systemctl restart mail4one.service
chown $appuser:$appuser "$certpath"
install -o "$appuser" -g "$appuser" -m 444 "$RENEWED_LINEAGE/fullchain.pem" -t "$certpath"
install -o "$appuser" -g "$appuser" -m 400 "$RENEWED_LINEAGE/privkey.pem" -t "$certpath"
systemctl restart $app.service
echo "$(date) Renewed and deployed certificates for $app" >> /var/log/cert-renew.log
fi fi

3
mail4one/__main__.py Normal file
View File

@ -0,0 +1,3 @@
from .server import main
main()

34
mail4one/config.py
View File

@ -1,8 +1,7 @@
"""Module for parsing mail4one config.json""" import json
import re import re
import logging import logging
from typing import Callable, Union, Optional from typing import Callable
from jata import Jata, MutableDefault from jata import Jata, MutableDefault
@ -42,48 +41,46 @@ class TLSCfg(Jata):
class ServerCfg(Jata): class ServerCfg(Jata):
server_type: str
host: str = "default" host: str = "default"
port: int port: int
# disabled: bool = False # disabled: bool = False
tls: Union[TLSCfg, str] = "default" tls: TLSCfg | str = "default"
class PopCfg(ServerCfg): class PopCfg(ServerCfg):
server_type = "pop"
port = 995 port = 995
timeout_seconds = 60 timeout_seconds = 60
class SmtpStartTLSCfg(ServerCfg): class SmtpStartTLSCfg(ServerCfg):
server_type = "smtp_starttls"
require_starttls = True
smtputf8 = True smtputf8 = True
port = 25 port = 25
class SmtpCfg(ServerCfg): class SmtpCfg(ServerCfg):
server_type = "smtp"
smtputf8 = True smtputf8 = True
port = 465 port = 465
class LogCfg(Jata): class LogCfg(Jata):
logfile = "CONSOLE" logfile = "STDOUT"
level = "INFO" level = "INFO"
class Config(Jata): class Config(Jata):
default_tls: Optional[TLSCfg] = None default_tls: TLSCfg | None
default_host: str = "0.0.0.0" default_host: str = "0.0.0.0"
logging: Optional[LogCfg] = None logging: LogCfg | None = None
mails_path: str mails_path: str
matches: list[Match] matches: list[Match]
boxes: list[Mbox] boxes: list[Mbox]
users: list[User] users: list[User]
servers: list[ServerCfg] pop: PopCfg | None = None
smtp_starttls: SmtpStartTLSCfg | None = None
smtp: SmtpCfg | None = None
# smtp_port_submission = 587
CheckerFn = Callable[[str], bool] CheckerFn = Callable[[str], bool]
@ -91,14 +88,17 @@ Checker = tuple[str, CheckerFn, bool]
def parse_checkers(cfg: Config) -> list[Checker]: def parse_checkers(cfg: Config) -> list[Checker]:
def make_match_fn(m: Match): def make_match_fn(m: Match):
if m.addrs and m.addr_rexs: if m.addrs and m.addr_rexs:
raise Exception("Both addrs and addr_rexs is set") raise Exception("Both addrs and addr_rexs is set")
if m.addrs: if m.addrs:
return lambda malias: malias in m.addrs return lambda malias: malias in m.addrs
if m.addr_rexs: elif m.addr_rexs:
compiled_res = [re.compile(reg) for reg in m.addr_rexs] compiled_res = [re.compile(reg) for reg in m.addr_rexs]
return lambda malias: any(reg.match(malias) for reg in compiled_res) return lambda malias: any(
reg.match(malias) for reg in compiled_res)
else:
raise Exception("Neither addrs nor addr_rexs is set") raise Exception("Neither addrs nor addr_rexs is set")
matches = {m.name: make_match_fn(Match(m)) for m in cfg.matches or []} matches = {m.name: make_match_fn(Match(m)) for m in cfg.matches or []}
@ -113,13 +113,13 @@ def parse_checkers(cfg: Config) -> list[Checker]:
return mbox_name, match_fn, rule.stop_check return mbox_name, match_fn, rule.stop_check
return [ return [
make_checker(mbox.name, Rule(rule)) make_checker(mbox.name, Rule(rule)) for mbox in cfg.boxes or []
for mbox in cfg.boxes or []
for rule in mbox.rules for rule in mbox.rules
] ]
def get_mboxes(addr: str, checks: list[Checker]) -> list[str]: def get_mboxes(addr: str, checks: list[Checker]) -> list[str]:
def inner(): def inner():
for mbox, match_fn, stop_check in checks: for mbox, match_fn, stop_check in checks:
if match_fn(addr): if match_fn(addr):

94
mail4one/pop3.py
View File

@ -2,15 +2,16 @@ import asyncio
import contextlib import contextlib
import contextvars import contextvars
import logging import logging
import os
import ssl import ssl
import random import uuid
from typing import Optional
from asyncio import StreamReader, StreamWriter
from dataclasses import dataclass from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path from pathlib import Path
from .config import User from .config import User
from .pwhash import parse_hash, check_pass, PWInfo from .pwhash import parse_hash, check_pass, PWInfo
from asyncio import StreamReader, StreamWriter
import random
from .poputils import ( from .poputils import (
InvalidCommand, InvalidCommand,
@ -26,7 +27,7 @@ from .poputils import (
end, end,
Request, Request,
MailEntry, MailEntry,
get_mail_fp, get_mail,
get_mails_list, get_mails_list,
MailList, MailList,
) )
@ -43,6 +44,7 @@ class State:
class SharedState: class SharedState:
def __init__(self, mails_path: Path, users: dict[str, tuple[PWInfo, str]]): def __init__(self, mails_path: Path, users: dict[str, tuple[PWInfo, str]]):
self.mails_path = mails_path self.mails_path = mails_path
self.users = users self.users = users
@ -54,7 +56,8 @@ class SharedState:
return self.counter return self.counter
c_shared_state: contextvars.ContextVar = contextvars.ContextVar("pop_shared_state") c_shared_state: contextvars.ContextVar = contextvars.ContextVar(
"pop_shared_state")
def scfg() -> SharedState: def scfg() -> SharedState:
@ -69,17 +72,18 @@ def state() -> State:
class PopLogger(logging.LoggerAdapter): class PopLogger(logging.LoggerAdapter):
def __init__(self): def __init__(self):
super().__init__(logging.getLogger("pop3"), None) super().__init__(logging.getLogger("pop3"), None)
def process(self, log_msg, kwargs): def process(self, msg, kwargs):
st: State = c_state.get(None) state: State = c_state.get(None)
if not st: if not state:
return super().process(log_msg, kwargs) return super().process(msg, kwargs)
user = "NA" user = "NA"
if st.username: if state.username:
user = st.username user = state.username
return super().process(f"{st.ip} {st.req_id} {user} {log_msg}", kwargs) return super().process(f"{state.ip} {state.req_id} {user} {msg}", kwargs)
logger = PopLogger() logger = PopLogger()
@ -101,6 +105,7 @@ async def next_req() -> Request:
if request.cmd == Command.QUIT: if request.cmd == Command.QUIT:
raise ClientQuit raise ClientQuit
return request return request
else:
raise ClientError(f"Bad command {InvalidCommand.RETRIES} times") raise ClientError(f"Bad command {InvalidCommand.RETRIES} times")
@ -112,8 +117,8 @@ async def expect_cmd(*commands: Command) -> Request:
return req return req
def write(data: bytes) -> None: def write(data) -> None:
logger.debug(f"Server: {data!r}") logger.debug(f"Server: {data}")
state().writer.write(data) state().writer.write(data)
@ -149,22 +154,24 @@ async def auth_stage() -> None:
write(ok("Following are supported")) write(ok("Following are supported"))
write(msg("USER")) write(msg("USER"))
write(end()) write(end())
continue else:
await handle_user_pass_auth(req) await handle_user_pass_auth(req)
if state().username in scfg().loggedin_users: if state().username in scfg().loggedin_users:
logger.warning( logger.warning(
f"User: {state().username} already has an active session" f"User: {state().username} already has an active session"
) )
raise AuthError("Already logged in") raise AuthError("Already logged in")
else:
scfg().loggedin_users.add(state().username) scfg().loggedin_users.add(state().username)
write(ok("Login successful")) write(ok("Login successful"))
return return
except AuthError as ae: except AuthError as ae:
write(err(f"Auth Failed: {ae}")) write(err(f"Auth Failed: {ae}"))
except ClientQuit: except ClientQuit as c:
write(ok("Bye")) write(ok("Bye"))
logger.warning("Client has QUIT before auth succeeded") logger.warning("Client has QUIT before auth succeeded")
raise raise
else:
raise ClientError("Failed to authenticate") raise ClientError("Failed to authenticate")
@ -211,12 +218,7 @@ def trans_command_retr(mails: MailList, req: Request) -> None:
entry = mails.get(req.arg1) entry = mails.get(req.arg1)
if entry: if entry:
write(ok("Contents follow")) write(ok("Contents follow"))
with get_mail_fp(entry) as fp: write(get_mail(entry))
for line in fp:
if line.startswith(b"."):
write(b".") # prepend dot
write(line)
# write(get_mail(entry)) # no prepend dot
write(end()) write(end())
mails.delete(req.arg1) mails.delete(req.arg1)
else: else:
@ -266,6 +268,7 @@ async def process_transactions(mails_list: list[MailEntry]) -> set[str]:
except KeyError: except KeyError:
write(err("Not implemented")) write(err("Not implemented"))
raise ClientError("We shouldn't reach here") raise ClientError("We shouldn't reach here")
else:
func(mails, req) func(mails, req)
await state().writer.drain() await state().writer.drain()
@ -277,7 +280,8 @@ def get_deleted_items(deleted_items_path: Path) -> set[str]:
return set() return set()
def save_deleted_items(deleted_items_path: Path, deleted_items: set[str]) -> None: def save_deleted_items(deleted_items_path: Path,
deleted_items: set[str]) -> None:
with deleted_items_path.open(mode="w") as f: with deleted_items_path.open(mode="w") as f:
f.writelines(f"{did}\n" for did in deleted_items) f.writelines(f"{did}\n" for did in deleted_items)
@ -294,11 +298,10 @@ async def transaction_stage() -> None:
new_deleted_items: set[str] = await process_transactions(mails_list) new_deleted_items: set[str] = await process_transactions(mails_list)
logger.info(f"completed transactions. Deleted:{len(new_deleted_items)}") logger.info(f"completed transactions. Deleted:{len(new_deleted_items)}")
if new_deleted_items: if new_deleted_items:
save_deleted_items( save_deleted_items(deleted_items_path,
deleted_items_path, existing_deleted_items.union(new_deleted_items) existing_deleted_items.union(new_deleted_items))
)
logger.info("Saved deleted items") logger.info(f"Saved deleted items")
async def start_session() -> None: async def start_session() -> None:
@ -309,15 +312,17 @@ async def start_session() -> None:
assert state().mbox assert state().mbox
await transaction_stage() await transaction_stage()
logger.info(f"User:{state().username} done") logger.info(f"User:{state().username} done")
except ClientDisconnected: except ClientDisconnected as c:
logger.info("Client disconnected") logger.info("Client disconnected")
pass
except ClientQuit: except ClientQuit:
logger.info("Client QUIT") logger.info("Client QUIT")
pass
except ClientError as c: except ClientError as c:
write(err("Something went wrong")) write(err("Something went wrong"))
logger.error(f"Unexpected client error: {c}") logger.error(f"Unexpected client error: {c}")
except: except Exception as e:
logger.exception("Serious client error") logger.error(f"Serious client error: {e}")
raise raise
finally: finally:
with contextlib.suppress(KeyError): with contextlib.suppress(KeyError):
@ -325,6 +330,7 @@ async def start_session() -> None:
def parse_users(users: list[User]) -> dict[str, tuple[PWInfo, str]]: def parse_users(users: list[User]) -> dict[str, tuple[PWInfo, str]]:
def inner(): def inner():
for user in users: for user in users:
user = User(user) user = User(user)
@ -334,22 +340,21 @@ def parse_users(users: list[User]) -> dict[str, tuple[PWInfo, str]]:
return dict(inner()) return dict(inner())
def make_pop_server_callback(mails_path: Path, users: list[User], timeout_seconds: int): def make_pop_server_callback(mails_path: Path, users: list[User],
s_state = SharedState(mails_path=mails_path, users=parse_users(users)) timeout_seconds: int):
scfg = SharedState(mails_path=mails_path, users=parse_users(users))
async def session_cb(reader: StreamReader, writer: StreamWriter): async def session_cb(reader: StreamReader, writer: StreamWriter):
c_shared_state.set(s_state) c_shared_state.set(scfg)
ip, _ = writer.get_extra_info("peername") ip, _ = writer.get_extra_info("peername")
c_state.set(State(reader=reader, writer=writer, ip=ip, req_id=s_state.next_id())) c_state.set(
logger.info("Got pop server callback") State(reader=reader, writer=writer, ip=ip, req_id=scfg.next_id()))
try: logger.info(f"Got pop server callback")
try: try:
return await asyncio.wait_for(start_session(), timeout_seconds) return await asyncio.wait_for(start_session(), timeout_seconds)
finally: finally:
writer.close() writer.close()
await writer.wait_closed() await writer.wait_closed()
except:
logger.exception("unexpected exception")
return session_cb return session_cb
@ -359,11 +364,11 @@ async def create_pop_server(
port: int, port: int,
mails_path: Path, mails_path: Path,
users: list[User], users: list[User],
ssl_context: Optional[ssl.SSLContext] = None, ssl_context: ssl.SSLContext | None = None,
timeout_seconds: int = 60, timeout_seconds: int = 60,
) -> asyncio.Server: ) -> asyncio.Server:
logging.info( logging.info(
f"Starting POP3 server {host=}, {port=}, {mails_path=!s}, {len(users)=}, {bool(ssl_context)=}, {timeout_seconds=}" f"Starting POP3 server {host=}, {port=}, {mails_path=}, {len(users)=}, {ssl_context != None=}, {timeout_seconds=}"
) )
return await asyncio.start_server( return await asyncio.start_server(
make_pop_server_callback(mails_path, users, timeout_seconds), make_pop_server_callback(mails_path, users, timeout_seconds),
@ -382,14 +387,13 @@ def debug_main():
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
import sys import sys
from .pwhash import gen_pwhash
_, mails_path, mbox = sys.argv _, mails_path, port, password = sys.argv
mails_path = Path(mails_path) mails_path = Path(mails_path)
users = [User(username="dummy", password_hash=gen_pwhash("dummy"), mbox=mbox)] port = int(port)
asyncio.run(a_main("127.0.0.1", 1101, mails_path, users=users)) asyncio.run(a_main(mails_path, port, password_hash=password_hash))
if __name__ == "__main__": if __name__ == "__main__":

12
mail4one/poputils.py
View File

@ -2,7 +2,6 @@ import os
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum, auto from enum import Enum, auto
from pathlib import Path from pathlib import Path
from contextlib import contextmanager
class ClientError(Exception): class ClientError(Exception):
@ -20,10 +19,12 @@ class ClientDisconnected(ClientError):
class InvalidCommand(ClientError): class InvalidCommand(ClientError):
RETRIES = 3 RETRIES = 3
"""WIll allow NUM_BAD_COMMANDS times""" """WIll allow NUM_BAD_COMMANDS times"""
pass
class AuthError(ClientError): class AuthError(ClientError):
RETRIES = 3 RETRIES = 3
pass
class Command(Enum): class Command(Enum):
@ -85,7 +86,7 @@ def parse_command(bline: bytes) -> Request:
if parts: if parts:
request.arg2, *parts = parts request.arg2, *parts = parts
if parts: if parts:
(request.rest,) = parts (request.rest, ) = parts
return request return request
@ -123,18 +124,13 @@ def set_nid(entries: list[MailEntry]):
entry.nid = i entry.nid = i
@contextmanager
def get_mail_fp(entry: MailEntry):
with open(entry.path, mode="rb") as fp:
yield fp
def get_mail(entry: MailEntry) -> bytes: def get_mail(entry: MailEntry) -> bytes:
with open(entry.path, mode="rb") as fp: with open(entry.path, mode="rb") as fp:
return fp.read() return fp.read()
class MailList: class MailList:
def __init__(self, entries: list[MailEntry]): def __init__(self, entries: list[MailEntry]):
self.entries = entries self.entries = entries
set_nid(self.entries) set_nid(self.entries)

15
mail4one/pwhash.py
View File

@ -19,13 +19,17 @@ KEY_LEN = 64 # This is python default
def gen_pwhash(password: str) -> str: def gen_pwhash(password: str) -> str:
salt = os.urandom(SALT_LEN) salt = os.urandom(SALT_LEN)
sh = scrypt( sh = scrypt(password.encode(),
password.encode(), salt=salt, n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P, dklen=KEY_LEN salt=salt,
) n=SCRYPT_N,
r=SCRYPT_R,
p=SCRYPT_P,
dklen=KEY_LEN)
return b32encode(VERSION + salt + sh).decode() return b32encode(VERSION + salt + sh).decode()
class PWInfo: class PWInfo:
def __init__(self, salt: bytes, sh: bytes): def __init__(self, salt: bytes, sh: bytes):
self.salt = salt self.salt = salt
self.scrypt_hash = sh self.scrypt_hash = sh
@ -36,13 +40,12 @@ def parse_hash(pwhash_str: str) -> PWInfo:
if not len(pwhash) == 1 + SALT_LEN + KEY_LEN: if not len(pwhash) == 1 + SALT_LEN + KEY_LEN:
raise Exception( raise Exception(
f"Invalid hash size, {len(pwhash)} != {1 + SALT_LEN + KEY_LEN}" f"Invalid hash size, {len(pwhash)} != {1 + SALT_LEN + KEY_LEN}")
)
if (ver := pwhash[0:1]) != VERSION: if (ver := pwhash[0:1]) != VERSION:
raise Exception(f"Invalid hash version, {ver!r} != {VERSION!r}") raise Exception(f"Invalid hash version, {ver!r} != {VERSION!r}")
salt, sh = pwhash[1 : SALT_LEN + 1], pwhash[-KEY_LEN:] salt, sh = pwhash[1:SALT_LEN + 1], pwhash[-KEY_LEN:]
return PWInfo(salt, sh) return PWInfo(salt, sh)

54
mail4one/server.py
View File

@ -1,14 +1,14 @@
import asyncio import asyncio
import logging import logging
import os
import ssl import ssl
import sys
from argparse import ArgumentParser from argparse import ArgumentParser
from pathlib import Path from pathlib import Path
from getpass import getpass from getpass import getpass
from typing import Optional, Union
from .smtp import create_smtp_server_starttls, create_smtp_server from .smtp import create_smtp_server_starttls, create_smtp_server
from .pop3 import create_pop_server from .pop3 import create_pop_server
from .version import VERSION
from . import config from . import config
from . import pwhash from . import pwhash
@ -21,47 +21,41 @@ def create_tls_context(certfile, keyfile) -> ssl.SSLContext:
def setup_logging(cfg: config.LogCfg): def setup_logging(cfg: config.LogCfg):
logging_format = ( logging_format = "%(asctime)s %(name)s %(levelname)s %(message)s @ %(filename)s:%(lineno)d"
"%(asctime)s %(name)s %(levelname)s %(message)s @ %(filename)s:%(lineno)d" if cfg.logfile == "STDOUT":
)
if cfg.logfile == "CONSOLE":
logging.basicConfig(level=cfg.level, format=logging_format) logging.basicConfig(level=cfg.level, format=logging_format)
else: else:
logging.basicConfig( logging.basicConfig(filename=cfg.logfile, level=cfg.level, format=logging_format)
filename=cfg.logfile, level=cfg.level, format=logging_format
)
async def a_main(cfg: config.Config) -> None: async def a_main(cfg: config.Config) -> None:
default_tls_context: Optional[ssl.SSLContext] = None default_tls_context: ssl.SSLContext | None = None
if tls := cfg.default_tls: if tls := cfg.default_tls:
logging.info(f"Initializing default tls {tls.certfile=}, {tls.keyfile=}") logging.info(f"Initializing default tls {tls.certfile=}, {tls.keyfile=}")
default_tls_context = create_tls_context(tls.certfile, tls.keyfile) default_tls_context = create_tls_context(tls.certfile, tls.keyfile)
def get_tls_context(tls: Union[config.TLSCfg, str]): def get_tls_context(tls: config.TLSCfg | str):
if tls == "default": if tls == "default":
return default_tls_context return default_tls_context
if tls == "disable": elif tls == "disable":
return None return None
tls_cfg = config.TLSCfg(tls) else:
tls_cfg = config.TLSCfg(pop.tls)
return create_tls_context(tls_cfg.certfile, tls_cfg.keyfile) return create_tls_context(tls_cfg.certfile, tls_cfg.keyfile)
def get_host(host): def get_host(host):
if host == "default": if host == "default":
return cfg.default_host return cfg.default_host
else:
return host return host
mbox_finder = config.gen_addr_to_mboxes(cfg) mbox_finder = config.gen_addr_to_mboxes(cfg)
servers: list[asyncio.Server] = [] servers: list[asyncio.Server] = []
if not cfg.servers: if cfg.pop:
logging.warning("Nothing to do!") pop = config.PopCfg(cfg.pop)
return
for scfg in cfg.servers:
if scfg.server_type == "pop":
pop = config.PopCfg(scfg)
pop_server = await create_pop_server( pop_server = await create_pop_server(
host=get_host(pop.host), host=get_host(pop.host),
port=pop.port, port=pop.port,
@ -71,8 +65,9 @@ async def a_main(cfg: config.Config) -> None:
timeout_seconds=pop.timeout_seconds, timeout_seconds=pop.timeout_seconds,
) )
servers.append(pop_server) servers.append(pop_server)
elif scfg.server_type == "smtp_starttls":
stls = config.SmtpStartTLSCfg(scfg) if cfg.smtp_starttls:
stls = config.SmtpStartTLSCfg(cfg.smtp_starttls)
stls_context = get_tls_context(stls.tls) stls_context = get_tls_context(stls.tls)
if not stls_context: if not stls_context:
raise Exception("starttls requires ssl_context") raise Exception("starttls requires ssl_context")
@ -82,28 +77,24 @@ async def a_main(cfg: config.Config) -> None:
mails_path=Path(cfg.mails_path), mails_path=Path(cfg.mails_path),
mbox_finder=mbox_finder, mbox_finder=mbox_finder,
ssl_context=stls_context, ssl_context=stls_context,
require_starttls=stls.require_starttls,
smtputf8=stls.smtputf8,
) )
servers.append(smtp_server_starttls) servers.append(smtp_server_starttls)
elif scfg.server_type == "smtp":
smtp = config.SmtpCfg(scfg) if cfg.smtp:
smtp = config.SmtpCfg(cfg.smtp)
smtp_server = await create_smtp_server( smtp_server = await create_smtp_server(
host=get_host(smtp.host), host=get_host(smtp.host),
port=smtp.port, port=smtp.port,
mails_path=Path(cfg.mails_path), mails_path=Path(cfg.mails_path),
mbox_finder=mbox_finder, mbox_finder=mbox_finder,
ssl_context=get_tls_context(smtp.tls), ssl_context=get_tls_context(smtp.tls),
smtputf8=smtp.smtputf8,
) )
servers.append(smtp_server) servers.append(smtp_server)
else:
logging.error(f"Unknown server {scfg.server_type=}")
if servers: if servers:
await asyncio.gather(*[server.serve_forever() for server in servers]) await asyncio.gather(*[server.serve_forever() for server in servers])
else: else:
logging.warning("Nothing to do!") logging.warn("Nothing to do!")
def main() -> None: def main() -> None:
@ -111,7 +102,6 @@ def main() -> None:
description="Personal Mail Server", description="Personal Mail Server",
epilog="See https://gitea.balki.me/balki/mail4one for more info", epilog="See https://gitea.balki.me/balki/mail4one for more info",
) )
parser.add_argument("-v", "--version", action="version", version=VERSION)
parser.add_argument( parser.add_argument(
"-e", "-e",
"--echo_password", "--echo_password",
@ -160,7 +150,7 @@ def main() -> None:
else: else:
cfg = config.Config(args.config.read_text()) cfg = config.Config(args.config.read_text())
setup_logging(config.LogCfg(cfg.logging)) setup_logging(config.LogCfg(cfg.logging))
logging.info(f"Starting mail4one {VERSION} {args.config=!s}") logging.info(f"Starting mail4one {args.config=}")
asyncio.run(a_main(cfg)) asyncio.run(a_main(cfg))

74
mail4one/smtp.py
View File

@ -1,18 +1,23 @@
import asyncio import asyncio
import io
import logging import logging
import mailbox
import ssl import ssl
import uuid import uuid
import shutil import shutil
from functools import partial from functools import partial
from pathlib import Path from pathlib import Path
from typing import Callable, Optional from typing import Callable
from . import config
from email.message import Message from email.message import Message
import email.policy import email.policy
from email.generator import BytesGenerator from email.generator import BytesGenerator
import tempfile import tempfile
import random
from aiosmtpd.handlers import AsyncMessage from aiosmtpd.handlers import Mailbox, AsyncMessage
from aiosmtpd.smtp import SMTP from aiosmtpd.smtp import SMTP, DATA_SIZE_DEFAULT
from aiosmtpd.smtp import SMTP as SMTPServer
from aiosmtpd.smtp import Envelope as SMTPEnvelope from aiosmtpd.smtp import Envelope as SMTPEnvelope
from aiosmtpd.smtp import Session as SMTPSession from aiosmtpd.smtp import Session as SMTPSession
@ -20,21 +25,20 @@ logger = logging.getLogger("smtp")
class MyHandler(AsyncMessage): class MyHandler(AsyncMessage):
def __init__(self, mails_path: Path, mbox_finder: Callable[[str], list[str]]):
def __init__(self, mails_path: Path, mbox_finder: Callable[[str],
list[str]]):
super().__init__() super().__init__()
self.mails_path = mails_path self.mails_path = mails_path
self.mbox_finder = mbox_finder self.mbox_finder = mbox_finder
self.rcpt_tos = []
self.peer = None
async def handle_DATA( async def handle_DATA(self, server: SMTPServer, session: SMTPSession,
self, server: SMTP, session: SMTPSession, envelope: SMTPEnvelope envelope: SMTPEnvelope) -> str:
) -> str:
self.rcpt_tos = envelope.rcpt_tos self.rcpt_tos = envelope.rcpt_tos
self.peer = session.peer self.peer = session.peer
return await super().handle_DATA(server, session, envelope) return await super().handle_DATA(server, session, envelope)
async def handle_message(self, message: Message): # type: ignore[override] async def handle_message(self, m: Message): # type: ignore[override]
all_mboxes: set[str] = set() all_mboxes: set[str] = set()
for addr in self.rcpt_tos: for addr in self.rcpt_tos:
for mbox in self.mbox_finder(addr.lower()): for mbox in self.mbox_finder(addr.lower()):
@ -51,7 +55,7 @@ class MyHandler(AsyncMessage):
temp_email_path = Path(tmpdir) / filename temp_email_path = Path(tmpdir) / filename
with open(temp_email_path, "wb") as fp: with open(temp_email_path, "wb") as fp:
gen = BytesGenerator(fp, policy=email.policy.SMTP) gen = BytesGenerator(fp, policy=email.policy.SMTP)
gen.flatten(message) gen.flatten(m)
for mbox in all_mboxes: for mbox in all_mboxes:
shutil.copy(temp_email_path, self.mails_path / mbox / "new") shutil.copy(temp_email_path, self.mails_path / mbox / "new")
logger.info( logger.info(
@ -59,37 +63,32 @@ class MyHandler(AsyncMessage):
) )
def protocol_factory_starttls( def protocol_factory_starttls(mails_path: Path,
mails_path: Path,
mbox_finder: Callable[[str], list[str]], mbox_finder: Callable[[str], list[str]],
context: ssl.SSLContext, context: ssl.SSLContext):
require_starttls: bool,
smtputf8: bool,
):
logger.info("Got smtp client cb starttls") logger.info("Got smtp client cb starttls")
try: try:
handler = MyHandler(mails_path, mbox_finder) handler = MyHandler(mails_path, mbox_finder)
smtp = SMTP( smtp = SMTP(
handler=handler, handler=handler,
require_starttls=require_starttls, require_starttls=True,
tls_context=context, tls_context=context,
enable_SMTPUTF8=smtputf8, enable_SMTPUTF8=True,
) )
except: except Exception as e:
logger.exception("Something went wrong") logger.error("Something went wrong", e)
raise raise
return smtp return smtp
def protocol_factory( def protocol_factory(mails_path: Path, mbox_finder: Callable[[str],
mails_path: Path, mbox_finder: Callable[[str], list[str]], smtputf8: bool list[str]]):
):
logger.info("Got smtp client cb") logger.info("Got smtp client cb")
try: try:
handler = MyHandler(mails_path, mbox_finder) handler = MyHandler(mails_path, mbox_finder)
smtp = SMTP(handler=handler, enable_SMTPUTF8=smtputf8) smtp = SMTP(handler=handler, enable_SMTPUTF8=True)
except: except Exception as e:
logger.exception("Something went wrong") logger.error("Something went wrong", e)
raise raise
return smtp return smtp
@ -100,22 +99,14 @@ async def create_smtp_server_starttls(
mails_path: Path, mails_path: Path,
mbox_finder: Callable[[str], list[str]], mbox_finder: Callable[[str], list[str]],
ssl_context: ssl.SSLContext, ssl_context: ssl.SSLContext,
require_starttls: bool,
smtputf8: bool,
) -> asyncio.Server: ) -> asyncio.Server:
logging.info( logging.info(
f"Starting SMTP STARTTLS server {host=}, {port=}, {mails_path=!s}, {bool(ssl_context)=}" f"Starting SMTP STARTTLS server {host=}, {port=}, {mails_path=}, {ssl_context != None=}"
) )
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
return await loop.create_server( return await loop.create_server(
partial( partial(protocol_factory_starttls, mails_path, mbox_finder,
protocol_factory_starttls, ssl_context),
mails_path,
mbox_finder,
ssl_context,
require_starttls,
smtputf8,
),
host=host, host=host,
port=port, port=port,
start_serving=False, start_serving=False,
@ -127,15 +118,14 @@ async def create_smtp_server(
port: int, port: int,
mails_path: Path, mails_path: Path,
mbox_finder: Callable[[str], list[str]], mbox_finder: Callable[[str], list[str]],
ssl_context: Optional[ssl.SSLContext], ssl_context: ssl.SSLContext | None = None,
smtputf8: bool,
) -> asyncio.Server: ) -> asyncio.Server:
logging.info( logging.info(
f"Starting SMTP server {host=}, {port=}, {mails_path=!s}, {bool(ssl_context)=}" f"Starting SMTP server {host=}, {port=}, {mails_path=}, {ssl_context != None=}"
) )
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
return await loop.create_server( return await loop.create_server(
partial(protocol_factory, mails_path, mbox_finder, smtputf8), partial(protocol_factory, mails_path, mbox_finder),
host=host, host=host,
port=port, port=port,
ssl=ssl_context, ssl=ssl_context,

1
mail4one/version.py
View File

@ -1 +0,0 @@
VERSION = "DEVELOMENT"

8
requirements.txt
View File

@ -1,5 +1,5 @@
-i https://pypi.org/simple -i https://pypi.org/simple
aiosmtpd==1.4.5; python_version >= '3.8' aiosmtpd==1.4.4.post2
atpublic==4.1.0; python_version >= '3.8' atpublic==4.0 ; python_version >= '3.8'
attrs==23.2.0; python_version >= '3.7' attrs==23.1.0 ; python_version >= '3.7'
python-jata==1.2; python_version >= '3.8' python-jata==1.2

3
run.py Normal file
View File

@ -0,0 +1,3 @@
from mail4one.server import main
main()

0
scripts/runtests.sh → runtests.sh
View File

29
scripts/get_version.sh
View File

@ -1,29 +0,0 @@
#!/bin/sh
commit=$(git rev-parse --short HEAD)
# This is true if there is a tag on current HEAD
if git describe --exact-match > /dev/null 2>&1
then
tag_val=$(git describe --dirty=DIRTY --exact-match)
case "$tag_val" in
*DIRTY)
echo "git-$commit-changes"
;;
v*) # Only consider tags starting with v
echo "$tag_val"
;;
*)
echo "git-$commit"
esac
else
tag_val=$(git describe --dirty=DIRTY)
case "$tag_val" in
*DIRTY)
echo "git-$commit-changes"
;;
*)
echo "git-$commit"
esac
fi

9
tests/test_config.py
View File

@ -67,11 +67,10 @@ class TestConfig(unittest.TestCase):
def test_get_mboxes(self) -> None: def test_get_mboxes(self) -> None:
cfg = config.Config(TEST_CONFIG) cfg = config.Config(TEST_CONFIG)
rules = config.parse_checkers(cfg) rules = config.parse_checkers(cfg)
self.assertEqual(config.get_mboxes("foo@bar.com", rules), ["spam"]) self.assertEqual(config.get_mboxes("foo@bar.com", rules), ['spam'])
self.assertEqual(config.get_mboxes("foo@mydomain.com", rules), ["all"]) self.assertEqual(config.get_mboxes("foo@mydomain.com", rules), ['all'])
self.assertEqual( self.assertEqual(config.get_mboxes("first.last@mydomain.com", rules),
config.get_mboxes("first.last@mydomain.com", rules), ["important", "all"] ['important', 'all'])
)
if __name__ == "__main__": if __name__ == "__main__":

88
tests/test_pop.py
View File

@ -4,29 +4,20 @@ import logging
import tempfile import tempfile
import time import time
import os import os
import poplib
from mail4one.pop3 import create_pop_server from mail4one.pop3 import create_pop_server
from mail4one.config import User from mail4one.config import User
from pathlib import Path from pathlib import Path
TEST_HASH = "".join( TEST_HASH = "".join(c for c in """
"""
AFTY5EVN7AX47ZL7UMH3BETYWFBTAV3XHR73CEFAJBPN2NIHPWD AFTY5EVN7AX47ZL7UMH3BETYWFBTAV3XHR73CEFAJBPN2NIHPWD
ZHV2UQSMSPHSQQ2A2BFQBNC77VL7F2UKATQNJZGYLCSU6C43UQD ZHV2UQSMSPHSQQ2A2BFQBNC77VL7F2UKATQNJZGYLCSU6C43UQD
AQXWXSWNGAEPGIMG2F3QDKBXL3MRHY6K2BPID64ZR6LABLPVSF AQXWXSWNGAEPGIMG2F3QDKBXL3MRHY6K2BPID64ZR6LABLPVSF
""".split() """ if not c.isspace())
)
TEST_USER = "foobar" TEST_USER = 'foobar'
TEST_MBOX = "foobar_mails" TEST_MBOX = 'foobar_mails'
TEST_USER2 = "foo2" USERS = [User(username=TEST_USER, password_hash=TEST_HASH, mbox=TEST_MBOX)]
TEST_MBOX2 = "foo2mails"
USERS = [
User(username=TEST_USER, password_hash=TEST_HASH, mbox=TEST_MBOX),
User(username=TEST_USER2, password_hash=TEST_HASH, mbox=TEST_MBOX2),
]
MAILS_PATH: Path MAILS_PATH: Path
@ -47,8 +38,7 @@ Hello bro\r
IlzVOJqu9Zp7twFAtzcV\r IlzVOJqu9Zp7twFAtzcV\r
yQVk36B0mGU2gtWxXLr\r yQVk36B0mGU2gtWxXLr\r
PeF0RtbI0mAuVPLQDHCi\r PeF0RtbI0mAuVPLQDHCi\r
\r \r\n"""
"""
def setUpModule() -> None: def setUpModule() -> None:
@ -57,21 +47,13 @@ def setUpModule() -> None:
td = tempfile.TemporaryDirectory(prefix="m41.pop.") td = tempfile.TemporaryDirectory(prefix="m41.pop.")
unittest.addModuleCleanup(td.cleanup) unittest.addModuleCleanup(td.cleanup)
MAILS_PATH = Path(td.name) MAILS_PATH = Path(td.name)
for mbox in (TEST_MBOX, TEST_MBOX2): os.mkdir(MAILS_PATH / TEST_MBOX)
os.mkdir(MAILS_PATH / mbox) for md in ('new', 'cur', 'tmp'):
for md in ("new", "cur", "tmp"): os.mkdir(MAILS_PATH / TEST_MBOX / md)
os.mkdir(MAILS_PATH / mbox / md) with open(MAILS_PATH / TEST_MBOX/ 'new/msg1.eml', 'wb') as f:
with open(MAILS_PATH / TEST_MBOX / "new/msg1.eml", "wb") as f:
f.write(TESTMAIL) f.write(TESTMAIL)
with open(MAILS_PATH / TEST_MBOX / "new/msg2.eml", "wb") as f: with open(MAILS_PATH / TEST_MBOX/ 'new/msg2.eml', 'wb') as f:
f.write(TESTMAIL) f.write(TESTMAIL)
with open(MAILS_PATH / TEST_MBOX2 / "new/msg1.eml", "wb") as f:
f.write(TESTMAIL)
f.write(b"More lines to follow\r\n")
f.write(b".Line starts with a dot\r\n")
f.write(b"some more lines\r\n")
f.write(b".\r\n")
f.write(b"Previous line just has a dot\r\n")
logging.debug(MAILS_PATH) logging.debug(MAILS_PATH)
@ -83,14 +65,13 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None: async def asyncSetUp(self) -> None:
logging.debug("at asyncSetUp") logging.debug("at asyncSetUp")
pop_server = await create_pop_server( pop_server = await create_pop_server(host='127.0.0.1',
host="127.0.0.1", port=7995, mails_path=MAILS_PATH, users=USERS port=7995,
) mails_path=MAILS_PATH,
users=USERS)
self.task = asyncio.create_task(pop_server.serve_forever()) self.task = asyncio.create_task(pop_server.serve_forever())
self.reader, self.writer = await asyncio.open_connection("127.0.0.1", 7995) self.reader, self.writer = await asyncio.open_connection(
'127.0.0.1', 7995)
# Additional writers to close
self.ws: list[asyncio.StreamWriter] = []
async def test_QUIT(self) -> None: async def test_QUIT(self) -> None:
dialog = """ dialog = """
@ -134,9 +115,8 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
await self.dialog_checker(dialog) await self.dialog_checker(dialog)
async def test_dupe_AUTH(self) -> None: async def test_dupe_AUTH(self) -> None:
r1, w1 = await asyncio.open_connection("127.0.0.1", 7995) r1, w1 = await asyncio.open_connection('127.0.0.1', 7995)
r2, w2 = await asyncio.open_connection("127.0.0.1", 7995) r2, w2 = await asyncio.open_connection('127.0.0.1', 7995)
self.ws += w1, w2
dialog = """ dialog = """
S: +OK Server Ready S: +OK Server Ready
C: USER foobar C: USER foobar
@ -217,36 +197,18 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
""" """
await self.dialog_checker(dialog) await self.dialog_checker(dialog)
async def test_poplib(self) -> None:
def run_poplib():
pc = poplib.POP3("127.0.0.1", 7995)
try:
self.assertEqual(b"+OK Server Ready", pc.getwelcome())
self.assertEqual(b"+OK Welcome", pc.user("foo2"))
self.assertEqual(b"+OK Login successful", pc.pass_("helloworld"))
_, eml, oc = pc.retr(1)
self.assertIn(b"Previous line just has a dot", eml)
self.assertIn(b".Line starts with a dot", eml)
self.assertIn(b".", eml)
finally:
pc.quit()
await asyncio.to_thread(run_poplib)
async def asyncTearDown(self) -> None: async def asyncTearDown(self) -> None:
logging.debug("at teardown") logging.debug("at teardown")
for w in self.ws + [self.writer]: self.writer.close()
w.close() await self.writer.wait_closed()
await w.wait_closed()
self.ws.clear()
self.task.cancel("test done") self.task.cancel("test done")
async def dialog_checker(self, dialog: str) -> None: async def dialog_checker(self, dialog: str) -> None:
await self.dialog_checker_impl(self.reader, self.writer, dialog) await self.dialog_checker_impl(self.reader, self.writer, dialog)
async def dialog_checker_impl( async def dialog_checker_impl(self, reader: asyncio.StreamReader,
self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, dialog: str writer: asyncio.StreamWriter,
) -> None: dialog: str) -> None:
for line in dialog.splitlines(): for line in dialog.splitlines():
line = line.strip() line = line.strip()
if not line: if not line:
@ -260,5 +222,5 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
self.assertEqual(data, resp) self.assertEqual(data, resp)
if __name__ == "__main__": if __name__ == '__main__':
unittest.main() unittest.main()

26
tests/test_pwhash.py
View File

@ -10,31 +10,27 @@ class TestPWHash(unittest.TestCase):
pwinfo = parse_hash(pwhash) pwinfo = parse_hash(pwhash)
self.assertEqual(len(pwinfo.salt), SALT_LEN) self.assertEqual(len(pwinfo.salt), SALT_LEN)
self.assertEqual(len(pwinfo.scrypt_hash), KEY_LEN) self.assertEqual(len(pwinfo.scrypt_hash), KEY_LEN)
self.assertTrue( self.assertTrue(check_pass(password, pwinfo),
check_pass(password, pwinfo), "check pass with correct password" "check pass with correct password")
) self.assertFalse(check_pass("foobar", pwinfo),
self.assertFalse(check_pass("foobar", pwinfo), "check pass with wrong password") "check pass with wrong password")
def test_hardcoded_hash(self): def test_hardcoded_hash(self):
test_hash = "".join( test_hash = "".join(c for c in """
c
for c in """
AFTY5EVN7AX47ZL7UMH3BETYWFBTAV3XHR73CEFAJBPN2NIHPWD AFTY5EVN7AX47ZL7UMH3BETYWFBTAV3XHR73CEFAJBPN2NIHPWD
ZHV2UQSMSPHSQQ2A2BFQBNC77VL7F2UKATQNJZGYLCSU6C43UQD ZHV2UQSMSPHSQQ2A2BFQBNC77VL7F2UKATQNJZGYLCSU6C43UQD
AQXWXSWNGAEPGIMG2F3QDKBXL3MRHY6K2BPID64ZR6LABLPVSF AQXWXSWNGAEPGIMG2F3QDKBXL3MRHY6K2BPID64ZR6LABLPVSF
""" """ if not c.isspace())
if not c.isspace()
)
pwinfo = parse_hash(test_hash) pwinfo = parse_hash(test_hash)
self.assertTrue( self.assertTrue(check_pass("helloworld", pwinfo),
check_pass("helloworld", pwinfo), "check pass with correct password" "check pass with correct password")
) self.assertFalse(check_pass("foobar", pwinfo),
self.assertFalse(check_pass("foobar", pwinfo), "check pass with wrong password") "check pass with wrong password")
def test_invalid_hash(self): def test_invalid_hash(self):
with self.assertRaises(Exception): with self.assertRaises(Exception):
parse_hash("sdlfkjdsklfjdsk") parse_hash("sdlfkjdsklfjdsk")
if __name__ == "__main__": if __name__ == '__main__':
unittest.main() unittest.main()

16
tests/test_smtp.py
View File

@ -10,7 +10,7 @@ from pathlib import Path
from mail4one.smtp import create_smtp_server from mail4one.smtp import create_smtp_server
TEST_MBOX = "foobar_mails" TEST_MBOX = 'foobar_mails'
MAILS_PATH: Path MAILS_PATH: Path
@ -21,7 +21,7 @@ def setUpModule() -> None:
unittest.addModuleCleanup(td.cleanup) unittest.addModuleCleanup(td.cleanup)
MAILS_PATH = Path(td.name) MAILS_PATH = Path(td.name)
os.mkdir(MAILS_PATH / TEST_MBOX) os.mkdir(MAILS_PATH / TEST_MBOX)
for md in ("new", "cur", "tmp"): for md in ('new', 'cur', 'tmp'):
os.mkdir(MAILS_PATH / TEST_MBOX / md) os.mkdir(MAILS_PATH / TEST_MBOX / md)
@ -32,10 +32,7 @@ class TestSMTP(unittest.IsolatedAsyncioTestCase):
host="127.0.0.1", host="127.0.0.1",
port=7996, port=7996,
mails_path=MAILS_PATH, mails_path=MAILS_PATH,
mbox_finder=lambda addr: [TEST_MBOX], mbox_finder=lambda addr: [TEST_MBOX])
ssl_context=None,
smtputf8=True,
)
self.task = asyncio.create_task(smtp_server.serve_forever()) self.task = asyncio.create_task(smtp_server.serve_forever())
async def test_send_mail(self) -> None: async def test_send_mail(self) -> None:
@ -48,9 +45,8 @@ class TestSMTP(unittest.IsolatedAsyncioTestCase):
msg = b"".join(l.strip() + b"\r\n" for l in msg.splitlines()) msg = b"".join(l.strip() + b"\r\n" for l in msg.splitlines())
def send_mail(): def send_mail():
with contextlib.closing( with contextlib.closing(smtplib.SMTP(host="127.0.0.1",
smtplib.SMTP(host="127.0.0.1", port=7996) port=7996)) as client:
) as client:
client.sendmail("foo@sender.com", "foo@bar.com", msg) client.sendmail("foo@sender.com", "foo@bar.com", msg)
_, local_port = client.sock.getsockname() _, local_port = client.sock.getsockname()
return local_port return local_port
@ -66,7 +62,7 @@ class TestSMTP(unittest.IsolatedAsyncioTestCase):
Byee Byee
""" """
expected = "".join(l.strip() + "\r\n" for l in expected.splitlines()) expected = "".join(l.strip() + "\r\n" for l in expected.splitlines())
mails = list((MAILS_PATH / TEST_MBOX / "new").glob("*")) mails = list((MAILS_PATH / TEST_MBOX / 'new').glob("*"))
self.assertEqual(len(mails), 1) self.assertEqual(len(mails), 1)
self.assertEqual(mails[0].read_bytes(), expected.encode()) self.assertEqual(mails[0].read_bytes(), expected.encode())