Compare commits

..

No commits in common. "main" and "v1.1" have entirely different histories.
main ... v1.1

9 changed files with 86 additions and 150 deletions

View File

@ -1,38 +0,0 @@
Notes for developers
## Running just one test
```
python -m unittest tests.test_pop.TestPop3.test_CAPA
```
## Patch for enable logging in test
Patch generated using below
```
git diff --patch -U1 tests >> ./DEVNOTES.md
```
Apply with below
```bash
git apply - <<PATCH
diff --git a/tests/test_pop.py b/tests/test_pop.py
index 55c1a91..a825665 100644
--- a/tests/test_pop.py
+++ b/tests/test_pop.py
@@ -55,3 +55,3 @@ def setUpModule() -> None:
global MAILS_PATH
- logging.basicConfig(level=logging.CRITICAL)
+ logging.basicConfig(level=logging.DEBUG)
td = tempfile.TemporaryDirectory(prefix="m41.pop.")
PATCH
```
## pylint
```
pylint mail4one/*py > /tmp/errs
vim +"cfile /tmp/errs"
```

View File

@ -1,10 +1,10 @@
# Needs python3 >= 3.9, sed, git for build
mail4one.pyz: requirements.txt mail4one/*py
# Needs python3 >= 3.9, sed, git for build, docker for tests
build: clean
python3 -m pip install -r requirements.txt --no-compile --target build
cp -r mail4one/ build/
sed -i "s/DEVELOMENT/$(shell scripts/get_version.sh)/" build/mail4one/version.py
find build -name "*.pyi" -o -name "py.typed" | xargs -I typefile rm typefile
rm -rf build/bin build/aiosmtpd/{docs,tests,qa}
rm -rf build/bin
rm -rf build/mail4one/__pycache__
rm -rf build/*.dist-info
python3 -m zipapp \
@ -13,19 +13,10 @@ mail4one.pyz: requirements.txt mail4one/*py
--main mail4one.server:main \
--compress build
.PHONY: build
build: clean mail4one.pyz
.PHONY: test
test: mail4one.pyz
PYTHONPATH=mail4one.pyz python3 -m unittest discover
.PHONY: clean
clean:
rm -rf build
rm -rf mail4one.pyz
.PHONY: docker-tests
docker-tests:
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.11-alpine sh scripts/runtests.sh
docker run --pull=always -v `pwd`:/app -w /app --rm python:3.10-alpine sh scripts/runtests.sh
@ -40,31 +31,24 @@ docker-tests:
requirements.txt: Pipfile.lock
pipenv requirements > requirements.txt
.PHONY: format
format:
black mail4one/*py tests/*py
.PHONY: build-dev
build-dev: requirements.txt build
.PHONY: setup
setup:
pipenv install
.PHONY: cleanup
cleanup:
pipenv --rm
.PHONY: update
update:
rm requirements.txt Pipfile.lock
pipenv update
pipenv requirements > requirements.txt
.PHONY: shell
shell:
MYPYPATH=$(shell ls -d `pipenv --venv`/lib/python3*/site-packages) pipenv shell
MYPYPATH=`pipenv --venv`/lib/python3.11/site-packages pipenv shell
.PHONY: dev-test
dev-test:
test:
pipenv run python -m unittest discover

View File

@ -1,5 +1,4 @@
"""Module for parsing mail4one config.json"""
import json
import re
import logging
from typing import Callable, Union, Optional
@ -57,14 +56,13 @@ class PopCfg(ServerCfg):
class SmtpStartTLSCfg(ServerCfg):
server_type = "smtp_starttls"
require_starttls = True
smtputf8 = True
smtputf8 = True # Not used yet
port = 25
class SmtpCfg(ServerCfg):
server_type = "smtp"
smtputf8 = True
server_type = "smtp_starttls"
smtputf8 = True # Not used yet
port = 465
@ -96,10 +94,11 @@ def parse_checkers(cfg: Config) -> list[Checker]:
raise Exception("Both addrs and addr_rexs is set")
if m.addrs:
return lambda malias: malias in m.addrs
if m.addr_rexs:
elif m.addr_rexs:
compiled_res = [re.compile(reg) for reg in m.addr_rexs]
return lambda malias: any(reg.match(malias) for reg in compiled_res)
raise Exception("Neither addrs nor addr_rexs is set")
else:
raise Exception("Neither addrs nor addr_rexs is set")
matches = {m.name: make_match_fn(Match(m)) for m in cfg.matches or []}
matches[DEFAULT_MATCH_ALL] = lambda _: True

View File

@ -2,15 +2,18 @@ import asyncio
import contextlib
import contextvars
import logging
import os
import ssl
import random
from typing import Optional
from asyncio import StreamReader, StreamWriter
import uuid
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from .config import User
from .pwhash import parse_hash, check_pass, PWInfo
from asyncio import StreamReader, StreamWriter
import random
from typing import Optional
from .poputils import (
InvalidCommand,
@ -72,14 +75,14 @@ class PopLogger(logging.LoggerAdapter):
def __init__(self):
super().__init__(logging.getLogger("pop3"), None)
def process(self, log_msg, kwargs):
st: State = c_state.get(None)
if not st:
return super().process(log_msg, kwargs)
def process(self, msg, kwargs):
state: State = c_state.get(None)
if not state:
return super().process(msg, kwargs)
user = "NA"
if st.username:
user = st.username
return super().process(f"{st.ip} {st.req_id} {user} {log_msg}", kwargs)
if state.username:
user = state.username
return super().process(f"{state.ip} {state.req_id} {user} {msg}", kwargs)
logger = PopLogger()
@ -101,7 +104,8 @@ async def next_req() -> Request:
if request.cmd == Command.QUIT:
raise ClientQuit
return request
raise ClientError(f"Bad command {InvalidCommand.RETRIES} times")
else:
raise ClientError(f"Bad command {InvalidCommand.RETRIES} times")
async def expect_cmd(*commands: Command) -> Request:
@ -149,23 +153,25 @@ async def auth_stage() -> None:
write(ok("Following are supported"))
write(msg("USER"))
write(end())
continue
await handle_user_pass_auth(req)
if state().username in scfg().loggedin_users:
logger.warning(
f"User: {state().username} already has an active session"
)
raise AuthError("Already logged in")
scfg().loggedin_users.add(state().username)
write(ok("Login successful"))
return
else:
await handle_user_pass_auth(req)
if state().username in scfg().loggedin_users:
logger.warning(
f"User: {state().username} already has an active session"
)
raise AuthError("Already logged in")
else:
scfg().loggedin_users.add(state().username)
write(ok("Login successful"))
return
except AuthError as ae:
write(err(f"Auth Failed: {ae}"))
except ClientQuit:
except ClientQuit as c:
write(ok("Bye"))
logger.warning("Client has QUIT before auth succeeded")
raise
raise ClientError("Failed to authenticate")
else:
raise ClientError("Failed to authenticate")
def trans_command_capa(_, __) -> None:
@ -266,8 +272,9 @@ async def process_transactions(mails_list: list[MailEntry]) -> set[str]:
except KeyError:
write(err("Not implemented"))
raise ClientError("We shouldn't reach here")
func(mails, req)
await state().writer.drain()
else:
func(mails, req)
await state().writer.drain()
def get_deleted_items(deleted_items_path: Path) -> set[str]:
@ -298,7 +305,7 @@ async def transaction_stage() -> None:
deleted_items_path, existing_deleted_items.union(new_deleted_items)
)
logger.info("Saved deleted items")
logger.info(f"Saved deleted items")
async def start_session() -> None:
@ -309,10 +316,12 @@ async def start_session() -> None:
assert state().mbox
await transaction_stage()
logger.info(f"User:{state().username} done")
except ClientDisconnected:
except ClientDisconnected as c:
logger.info("Client disconnected")
pass
except ClientQuit:
logger.info("Client QUIT")
pass
except ClientError as c:
write(err("Something went wrong"))
logger.error(f"Unexpected client error: {c}")
@ -335,13 +344,13 @@ def parse_users(users: list[User]) -> dict[str, tuple[PWInfo, str]]:
def make_pop_server_callback(mails_path: Path, users: list[User], timeout_seconds: int):
s_state = SharedState(mails_path=mails_path, users=parse_users(users))
scfg = SharedState(mails_path=mails_path, users=parse_users(users))
async def session_cb(reader: StreamReader, writer: StreamWriter):
c_shared_state.set(s_state)
c_shared_state.set(scfg)
ip, _ = writer.get_extra_info("peername")
c_state.set(State(reader=reader, writer=writer, ip=ip, req_id=s_state.next_id()))
logger.info("Got pop server callback")
c_state.set(State(reader=reader, writer=writer, ip=ip, req_id=scfg.next_id()))
logger.info(f"Got pop server callback")
try:
try:
return await asyncio.wait_for(start_session(), timeout_seconds)
@ -363,7 +372,7 @@ async def create_pop_server(
timeout_seconds: int = 60,
) -> asyncio.Server:
logging.info(
f"Starting POP3 server {host=}, {port=}, {mails_path=!s}, {len(users)=}, {bool(ssl_context)=}, {timeout_seconds=}"
f"Starting POP3 server {host=}, {port=}, {mails_path=!s}, {len(users)=}, {ssl_context != None=}, {timeout_seconds=}"
)
return await asyncio.start_server(
make_pop_server_callback(mails_path, users, timeout_seconds),

View File

@ -20,10 +20,12 @@ class ClientDisconnected(ClientError):
class InvalidCommand(ClientError):
RETRIES = 3
"""WIll allow NUM_BAD_COMMANDS times"""
pass
class AuthError(ClientError):
RETRIES = 3
pass
class Command(Enum):

View File

@ -1,10 +1,11 @@
import asyncio
import logging
import os
import ssl
import sys
from argparse import ArgumentParser
from pathlib import Path
from getpass import getpass
from typing import Optional, Union
from .smtp import create_smtp_server_starttls, create_smtp_server
from .pop3 import create_pop_server
@ -12,6 +13,7 @@ from .version import VERSION
from . import config
from . import pwhash
from typing import Optional, Union
def create_tls_context(certfile, keyfile) -> ssl.SSLContext:
@ -42,15 +44,17 @@ async def a_main(cfg: config.Config) -> None:
def get_tls_context(tls: Union[config.TLSCfg, str]):
if tls == "default":
return default_tls_context
if tls == "disable":
elif tls == "disable":
return None
tls_cfg = config.TLSCfg(tls)
return create_tls_context(tls_cfg.certfile, tls_cfg.keyfile)
else:
tls_cfg = config.TLSCfg(tls)
return create_tls_context(tls_cfg.certfile, tls_cfg.keyfile)
def get_host(host):
if host == "default":
return cfg.default_host
return host
else:
return host
mbox_finder = config.gen_addr_to_mboxes(cfg)
servers: list[asyncio.Server] = []
@ -82,8 +86,6 @@ async def a_main(cfg: config.Config) -> None:
mails_path=Path(cfg.mails_path),
mbox_finder=mbox_finder,
ssl_context=stls_context,
require_starttls=stls.require_starttls,
smtputf8=stls.smtputf8,
)
servers.append(smtp_server_starttls)
elif scfg.server_type == "smtp":
@ -94,7 +96,6 @@ async def a_main(cfg: config.Config) -> None:
mails_path=Path(cfg.mails_path),
mbox_finder=mbox_finder,
ssl_context=get_tls_context(smtp.tls),
smtputf8=smtp.smtputf8,
)
servers.append(smtp_server)
else:

View File

@ -1,18 +1,23 @@
import asyncio
import io
import logging
import mailbox
import ssl
import uuid
import shutil
from functools import partial
from pathlib import Path
from typing import Callable, Optional
from . import config
from email.message import Message
import email.policy
from email.generator import BytesGenerator
import tempfile
import random
from aiosmtpd.handlers import AsyncMessage
from aiosmtpd.smtp import SMTP
from aiosmtpd.handlers import Mailbox, AsyncMessage
from aiosmtpd.smtp import SMTP, DATA_SIZE_DEFAULT
from aiosmtpd.smtp import SMTP as SMTPServer
from aiosmtpd.smtp import Envelope as SMTPEnvelope
from aiosmtpd.smtp import Session as SMTPSession
@ -24,17 +29,15 @@ class MyHandler(AsyncMessage):
super().__init__()
self.mails_path = mails_path
self.mbox_finder = mbox_finder
self.rcpt_tos = []
self.peer = None
async def handle_DATA(
self, server: SMTP, session: SMTPSession, envelope: SMTPEnvelope
self, server: SMTPServer, session: SMTPSession, envelope: SMTPEnvelope
) -> str:
self.rcpt_tos = envelope.rcpt_tos
self.peer = session.peer
return await super().handle_DATA(server, session, envelope)
async def handle_message(self, message: Message): # type: ignore[override]
async def handle_message(self, m: Message): # type: ignore[override]
all_mboxes: set[str] = set()
for addr in self.rcpt_tos:
for mbox in self.mbox_finder(addr.lower()):
@ -51,7 +54,7 @@ class MyHandler(AsyncMessage):
temp_email_path = Path(tmpdir) / filename
with open(temp_email_path, "wb") as fp:
gen = BytesGenerator(fp, policy=email.policy.SMTP)
gen.flatten(message)
gen.flatten(m)
for mbox in all_mboxes:
shutil.copy(temp_email_path, self.mails_path / mbox / "new")
logger.info(
@ -60,20 +63,16 @@ class MyHandler(AsyncMessage):
def protocol_factory_starttls(
mails_path: Path,
mbox_finder: Callable[[str], list[str]],
context: ssl.SSLContext,
require_starttls: bool,
smtputf8: bool,
mails_path: Path, mbox_finder: Callable[[str], list[str]], context: ssl.SSLContext
):
logger.info("Got smtp client cb starttls")
try:
handler = MyHandler(mails_path, mbox_finder)
smtp = SMTP(
handler=handler,
require_starttls=require_starttls,
require_starttls=True,
tls_context=context,
enable_SMTPUTF8=smtputf8,
enable_SMTPUTF8=True,
)
except:
logger.exception("Something went wrong")
@ -81,13 +80,11 @@ def protocol_factory_starttls(
return smtp
def protocol_factory(
mails_path: Path, mbox_finder: Callable[[str], list[str]], smtputf8: bool
):
def protocol_factory(mails_path: Path, mbox_finder: Callable[[str], list[str]]):
logger.info("Got smtp client cb")
try:
handler = MyHandler(mails_path, mbox_finder)
smtp = SMTP(handler=handler, enable_SMTPUTF8=smtputf8)
smtp = SMTP(handler=handler, enable_SMTPUTF8=True)
except:
logger.exception("Something went wrong")
raise
@ -100,22 +97,13 @@ async def create_smtp_server_starttls(
mails_path: Path,
mbox_finder: Callable[[str], list[str]],
ssl_context: ssl.SSLContext,
require_starttls: bool,
smtputf8: bool,
) -> asyncio.Server:
logging.info(
f"Starting SMTP STARTTLS server {host=}, {port=}, {mails_path=!s}, {bool(ssl_context)=}"
f"Starting SMTP STARTTLS server {host=}, {port=}, {mails_path=!s}, {ssl_context != None=}"
)
loop = asyncio.get_event_loop()
return await loop.create_server(
partial(
protocol_factory_starttls,
mails_path,
mbox_finder,
ssl_context,
require_starttls,
smtputf8,
),
partial(protocol_factory_starttls, mails_path, mbox_finder, ssl_context),
host=host,
port=port,
start_serving=False,
@ -127,15 +115,14 @@ async def create_smtp_server(
port: int,
mails_path: Path,
mbox_finder: Callable[[str], list[str]],
ssl_context: Optional[ssl.SSLContext],
smtputf8: bool,
ssl_context: Optional[ssl.SSLContext] = None,
) -> asyncio.Server:
logging.info(
f"Starting SMTP server {host=}, {port=}, {mails_path=!s}, {bool(ssl_context)=}"
f"Starting SMTP server {host=}, {port=}, {mails_path=!s}, {ssl_context != None=}"
)
loop = asyncio.get_event_loop()
return await loop.create_server(
partial(protocol_factory, mails_path, mbox_finder, smtputf8),
partial(protocol_factory, mails_path, mbox_finder),
host=host,
port=port,
ssl=ssl_context,

View File

@ -89,9 +89,6 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
self.task = asyncio.create_task(pop_server.serve_forever())
self.reader, self.writer = await asyncio.open_connection("127.0.0.1", 7995)
# Additional writers to close
self.ws: list[asyncio.StreamWriter] = []
async def test_QUIT(self) -> None:
dialog = """
S: +OK Server Ready
@ -136,7 +133,6 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
async def test_dupe_AUTH(self) -> None:
r1, w1 = await asyncio.open_connection("127.0.0.1", 7995)
r2, w2 = await asyncio.open_connection("127.0.0.1", 7995)
self.ws += w1, w2
dialog = """
S: +OK Server Ready
C: USER foobar
@ -235,10 +231,8 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
async def asyncTearDown(self) -> None:
logging.debug("at teardown")
for w in self.ws + [self.writer]:
w.close()
await w.wait_closed()
self.ws.clear()
self.writer.close()
await self.writer.wait_closed()
self.task.cancel("test done")
async def dialog_checker(self, dialog: str) -> None:

View File

@ -33,8 +33,6 @@ class TestSMTP(unittest.IsolatedAsyncioTestCase):
port=7996,
mails_path=MAILS_PATH,
mbox_finder=lambda addr: [TEST_MBOX],
ssl_context=None,
smtputf8=True,
)
self.task = asyncio.create_task(smtp_server.serve_forever())