DEVNOTES.md

@@ -1,38 +0,0 @@
-Notes for developers
-
-## Running just one test
-
-```
-python -m unittest tests.test_pop.TestPop3.test_CAPA
-```
-
-## Patch for enable logging in test
-
-Patch generated using below
-
-```
-git diff --patch -U1 tests >> ./DEVNOTES.md
-```
-
-Apply with below
-
-```bash
-git apply - <<PATCH
-diff --git a/tests/test_pop.py b/tests/test_pop.py
-index 55c1a91..a825665 100644
---- a/tests/test_pop.py
-+++ b/tests/test_pop.py
-@@ -55,3 +55,3 @@ def setUpModule() -> None:
-     global MAILS_PATH
--    logging.basicConfig(level=logging.CRITICAL)
-+    logging.basicConfig(level=logging.DEBUG)
-     td = tempfile.TemporaryDirectory(prefix="m41.pop.")
-PATCH
-```
-
-## pylint
-
-```
-pylint mail4one/*py > /tmp/errs
-vim +"cfile /tmp/errs"
-```

Makefile

@@ -1,10 +1,10 @@
-# Needs python3 >= 3.9, sed, git for build
-mail4one.pyz: requirements.txt mail4one/*py
+# Needs python3 >= 3.9, sed, git for build, docker for tests
+build: clean
 	python3 -m pip install -r requirements.txt --no-compile --target build
 	cp -r mail4one/ build/
 	sed -i "s/DEVELOMENT/$(shell scripts/get_version.sh)/" build/mail4one/version.py
 	find build -name "*.pyi" -o -name "py.typed" | xargs -I typefile rm typefile
-	rm -rf build/bin build/aiosmtpd/{docs,tests,qa}
+	rm -rf build/bin
 	rm -rf build/mail4one/__pycache__
 	rm -rf build/*.dist-info
 	python3 -m zipapp \
@@ -13,19 +13,10 @@ mail4one.pyz: requirements.txt mail4one/*py
 		--main mail4one.server:main \
 		--compress build
 
-.PHONY: build
-build: clean mail4one.pyz
-
-.PHONY: test
-test: mail4one.pyz
-	PYTHONPATH=mail4one.pyz python3 -m unittest discover
-
-.PHONY: clean
 clean:
 	rm -rf build
 	rm -rf mail4one.pyz
 
-.PHONY: docker-tests
 docker-tests:
 	docker run --pull=always -v `pwd`:/app -w /app --rm python:3.11-alpine sh scripts/runtests.sh
 	docker run --pull=always -v `pwd`:/app -w /app --rm python:3.10-alpine sh scripts/runtests.sh
@@ -40,31 +31,24 @@ docker-tests:
 requirements.txt: Pipfile.lock
 	pipenv requirements > requirements.txt
 
-.PHONY: format
 format:
 	black mail4one/*py tests/*py
 
-.PHONY: build-dev
 build-dev: requirements.txt build
 
-.PHONY: setup
 setup:
 	pipenv install
 
-.PHONY: cleanup
 cleanup:
 	pipenv --rm
 
-.PHONY: update
 update:
 	rm requirements.txt Pipfile.lock
 	pipenv update
 	pipenv requirements > requirements.txt
 
-.PHONY: shell
 shell:
-	MYPYPATH=$(shell ls -d `pipenv --venv`/lib/python3*/site-packages) pipenv shell
+	MYPYPATH=`pipenv --venv`/lib/python3.11/site-packages pipenv shell
 	
-.PHONY: dev-test
-dev-test:
+test:
 	pipenv run python -m unittest discover

mail4one/config.py

@@ -1,5 +1,4 @@
-"""Module for parsing mail4one config.json"""
-
+import json
 import re
 import logging
 from typing import Callable, Union, Optional
@@ -57,14 +56,13 @@ class PopCfg(ServerCfg):
 
 class SmtpStartTLSCfg(ServerCfg):
     server_type = "smtp_starttls"
-    require_starttls = True
-    smtputf8 = True
+    smtputf8 = True  # Not used yet
     port = 25
 
 
 class SmtpCfg(ServerCfg):
-    server_type = "smtp"
-    smtputf8 = True
+    server_type = "smtp_starttls"
+    smtputf8 = True  # Not used yet
     port = 465
 
 
@@ -96,10 +94,11 @@ def parse_checkers(cfg: Config) -> list[Checker]:
             raise Exception("Both addrs and addr_rexs is set")
         if m.addrs:
             return lambda malias: malias in m.addrs
-        if m.addr_rexs:
+        elif m.addr_rexs:
             compiled_res = [re.compile(reg) for reg in m.addr_rexs]
             return lambda malias: any(reg.match(malias) for reg in compiled_res)
-        raise Exception("Neither addrs nor addr_rexs is set")
+        else:
+            raise Exception("Neither addrs nor addr_rexs is set")
 
     matches = {m.name: make_match_fn(Match(m)) for m in cfg.matches or []}
     matches[DEFAULT_MATCH_ALL] = lambda _: True

mail4one/pop3.py

@@ -2,15 +2,18 @@ import asyncio
 import contextlib
 import contextvars
 import logging
+import os
 import ssl
-import random
-from typing import Optional
-from asyncio import StreamReader, StreamWriter
+import uuid
 from dataclasses import dataclass
+from hashlib import sha256
 from pathlib import Path
 from .config import User
 from .pwhash import parse_hash, check_pass, PWInfo
+from asyncio import StreamReader, StreamWriter
+import random
 
+from typing import Optional
 
 from .poputils import (
     InvalidCommand,
@@ -72,14 +75,14 @@ class PopLogger(logging.LoggerAdapter):
     def __init__(self):
         super().__init__(logging.getLogger("pop3"), None)
 
-    def process(self, log_msg, kwargs):
-        st: State = c_state.get(None)
-        if not st:
-            return super().process(log_msg, kwargs)
+    def process(self, msg, kwargs):
+        state: State = c_state.get(None)
+        if not state:
+            return super().process(msg, kwargs)
         user = "NA"
-        if st.username:
-            user = st.username
-        return super().process(f"{st.ip} {st.req_id} {user} {log_msg}", kwargs)
+        if state.username:
+            user = state.username
+        return super().process(f"{state.ip} {state.req_id} {user} {msg}", kwargs)
 
 
 logger = PopLogger()
@@ -101,7 +104,8 @@ async def next_req() -> Request:
             if request.cmd == Command.QUIT:
                 raise ClientQuit
             return request
-    raise ClientError(f"Bad command {InvalidCommand.RETRIES} times")
+    else:
+        raise ClientError(f"Bad command {InvalidCommand.RETRIES} times")
 
 
 async def expect_cmd(*commands: Command) -> Request:
@@ -149,23 +153,25 @@ async def auth_stage() -> None:
                 write(ok("Following are supported"))
                 write(msg("USER"))
                 write(end())
-                continue
-            await handle_user_pass_auth(req)
-            if state().username in scfg().loggedin_users:
-                logger.warning(
-                    f"User: {state().username} already has an active session"
-                )
-                raise AuthError("Already logged in")
-            scfg().loggedin_users.add(state().username)
-            write(ok("Login successful"))
-            return
+            else:
+                await handle_user_pass_auth(req)
+                if state().username in scfg().loggedin_users:
+                    logger.warning(
+                        f"User: {state().username} already has an active session"
+                    )
+                    raise AuthError("Already logged in")
+                else:
+                    scfg().loggedin_users.add(state().username)
+                    write(ok("Login successful"))
+                    return
         except AuthError as ae:
             write(err(f"Auth Failed: {ae}"))
-        except ClientQuit:
+        except ClientQuit as c:
             write(ok("Bye"))
             logger.warning("Client has QUIT before auth succeeded")
             raise
-    raise ClientError("Failed to authenticate")
+    else:
+        raise ClientError("Failed to authenticate")
 
 
 def trans_command_capa(_, __) -> None:
@@ -266,8 +272,9 @@ async def process_transactions(mails_list: list[MailEntry]) -> set[str]:
         except KeyError:
             write(err("Not implemented"))
             raise ClientError("We shouldn't reach here")
-        func(mails, req)
-        await state().writer.drain()
+        else:
+            func(mails, req)
+            await state().writer.drain()
 
 
 def get_deleted_items(deleted_items_path: Path) -> set[str]:
@@ -298,7 +305,7 @@ async def transaction_stage() -> None:
             deleted_items_path, existing_deleted_items.union(new_deleted_items)
         )
 
-    logger.info("Saved deleted items")
+    logger.info(f"Saved deleted items")
 
 
 async def start_session() -> None:
@@ -309,10 +316,12 @@ async def start_session() -> None:
         assert state().mbox
         await transaction_stage()
         logger.info(f"User:{state().username} done")
-    except ClientDisconnected:
+    except ClientDisconnected as c:
         logger.info("Client disconnected")
+        pass
     except ClientQuit:
         logger.info("Client QUIT")
+        pass
     except ClientError as c:
         write(err("Something went wrong"))
         logger.error(f"Unexpected client error: {c}")
@@ -335,13 +344,13 @@ def parse_users(users: list[User]) -> dict[str, tuple[PWInfo, str]]:
 
 
 def make_pop_server_callback(mails_path: Path, users: list[User], timeout_seconds: int):
-    s_state = SharedState(mails_path=mails_path, users=parse_users(users))
+    scfg = SharedState(mails_path=mails_path, users=parse_users(users))
 
     async def session_cb(reader: StreamReader, writer: StreamWriter):
-        c_shared_state.set(s_state)
+        c_shared_state.set(scfg)
         ip, _ = writer.get_extra_info("peername")
-        c_state.set(State(reader=reader, writer=writer, ip=ip, req_id=s_state.next_id()))
-        logger.info("Got pop server callback")
+        c_state.set(State(reader=reader, writer=writer, ip=ip, req_id=scfg.next_id()))
+        logger.info(f"Got pop server callback")
         try:
             try:
                 return await asyncio.wait_for(start_session(), timeout_seconds)
@@ -363,7 +372,7 @@ async def create_pop_server(
     timeout_seconds: int = 60,
 ) -> asyncio.Server:
     logging.info(
-        f"Starting POP3 server {host=}, {port=}, {mails_path=!s}, {len(users)=}, {bool(ssl_context)=}, {timeout_seconds=}"
+        f"Starting POP3 server {host=}, {port=}, {mails_path=!s}, {len(users)=}, {ssl_context != None=}, {timeout_seconds=}"
     )
     return await asyncio.start_server(
         make_pop_server_callback(mails_path, users, timeout_seconds),

mail4one/poputils.py

@@ -20,10 +20,12 @@ class ClientDisconnected(ClientError):
 class InvalidCommand(ClientError):
     RETRIES = 3
     """WIll allow NUM_BAD_COMMANDS times"""
+    pass
 
 
 class AuthError(ClientError):
     RETRIES = 3
+    pass
 
 
 class Command(Enum):

mail4one/server.py

@@ -1,10 +1,11 @@
 import asyncio
 import logging
+import os
 import ssl
+import sys
 from argparse import ArgumentParser
 from pathlib import Path
 from getpass import getpass
-from typing import Optional, Union
 
 from .smtp import create_smtp_server_starttls, create_smtp_server
 from .pop3 import create_pop_server
@@ -12,6 +13,7 @@ from .version import VERSION
 
 from . import config
 from . import pwhash
+from typing import Optional, Union
 
 
 def create_tls_context(certfile, keyfile) -> ssl.SSLContext:
@@ -42,15 +44,17 @@ async def a_main(cfg: config.Config) -> None:
     def get_tls_context(tls: Union[config.TLSCfg, str]):
         if tls == "default":
             return default_tls_context
-        if tls == "disable":
+        elif tls == "disable":
             return None
-        tls_cfg = config.TLSCfg(tls)
-        return create_tls_context(tls_cfg.certfile, tls_cfg.keyfile)
+        else:
+            tls_cfg = config.TLSCfg(tls)
+            return create_tls_context(tls_cfg.certfile, tls_cfg.keyfile)
 
     def get_host(host):
         if host == "default":
             return cfg.default_host
-        return host
+        else:
+            return host
 
     mbox_finder = config.gen_addr_to_mboxes(cfg)
     servers: list[asyncio.Server] = []
@@ -82,8 +86,6 @@ async def a_main(cfg: config.Config) -> None:
                 mails_path=Path(cfg.mails_path),
                 mbox_finder=mbox_finder,
                 ssl_context=stls_context,
-                require_starttls=stls.require_starttls,
-                smtputf8=stls.smtputf8,
             )
             servers.append(smtp_server_starttls)
         elif scfg.server_type == "smtp":
@@ -94,7 +96,6 @@ async def a_main(cfg: config.Config) -> None:
                 mails_path=Path(cfg.mails_path),
                 mbox_finder=mbox_finder,
                 ssl_context=get_tls_context(smtp.tls),
-                smtputf8=smtp.smtputf8,
             )
             servers.append(smtp_server)
         else:

mail4one/smtp.py

@@ -1,18 +1,23 @@
 import asyncio
+import io
 import logging
+import mailbox
 import ssl
 import uuid
 import shutil
 from functools import partial
 from pathlib import Path
 from typing import Callable, Optional
+from . import config
 from email.message import Message
 import email.policy
 from email.generator import BytesGenerator
 import tempfile
+import random
 
-from aiosmtpd.handlers import AsyncMessage
-from aiosmtpd.smtp import SMTP
+from aiosmtpd.handlers import Mailbox, AsyncMessage
+from aiosmtpd.smtp import SMTP, DATA_SIZE_DEFAULT
+from aiosmtpd.smtp import SMTP as SMTPServer
 from aiosmtpd.smtp import Envelope as SMTPEnvelope
 from aiosmtpd.smtp import Session as SMTPSession
 
@@ -24,17 +29,15 @@ class MyHandler(AsyncMessage):
         super().__init__()
         self.mails_path = mails_path
         self.mbox_finder = mbox_finder
-        self.rcpt_tos = []
-        self.peer = None
 
     async def handle_DATA(
-        self, server: SMTP, session: SMTPSession, envelope: SMTPEnvelope
+        self, server: SMTPServer, session: SMTPSession, envelope: SMTPEnvelope
     ) -> str:
         self.rcpt_tos = envelope.rcpt_tos
         self.peer = session.peer
         return await super().handle_DATA(server, session, envelope)
 
-    async def handle_message(self, message: Message):  # type: ignore[override]
+    async def handle_message(self, m: Message):  # type: ignore[override]
         all_mboxes: set[str] = set()
         for addr in self.rcpt_tos:
             for mbox in self.mbox_finder(addr.lower()):
@@ -51,7 +54,7 @@ class MyHandler(AsyncMessage):
             temp_email_path = Path(tmpdir) / filename
             with open(temp_email_path, "wb") as fp:
                 gen = BytesGenerator(fp, policy=email.policy.SMTP)
-                gen.flatten(message)
+                gen.flatten(m)
             for mbox in all_mboxes:
                 shutil.copy(temp_email_path, self.mails_path / mbox / "new")
             logger.info(
@@ -60,20 +63,16 @@ class MyHandler(AsyncMessage):
 
 
 def protocol_factory_starttls(
-    mails_path: Path,
-    mbox_finder: Callable[[str], list[str]],
-    context: ssl.SSLContext,
-    require_starttls: bool,
-    smtputf8: bool,
+    mails_path: Path, mbox_finder: Callable[[str], list[str]], context: ssl.SSLContext
 ):
     logger.info("Got smtp client cb starttls")
     try:
         handler = MyHandler(mails_path, mbox_finder)
         smtp = SMTP(
             handler=handler,
-            require_starttls=require_starttls,
+            require_starttls=True,
             tls_context=context,
-            enable_SMTPUTF8=smtputf8,
+            enable_SMTPUTF8=True,
         )
     except:
         logger.exception("Something went wrong")
@@ -81,13 +80,11 @@ def protocol_factory_starttls(
     return smtp
 
 
-def protocol_factory(
-    mails_path: Path, mbox_finder: Callable[[str], list[str]], smtputf8: bool
-):
+def protocol_factory(mails_path: Path, mbox_finder: Callable[[str], list[str]]):
     logger.info("Got smtp client cb")
     try:
         handler = MyHandler(mails_path, mbox_finder)
-        smtp = SMTP(handler=handler, enable_SMTPUTF8=smtputf8)
+        smtp = SMTP(handler=handler, enable_SMTPUTF8=True)
     except:
         logger.exception("Something went wrong")
         raise
@@ -100,22 +97,13 @@ async def create_smtp_server_starttls(
     mails_path: Path,
     mbox_finder: Callable[[str], list[str]],
     ssl_context: ssl.SSLContext,
-    require_starttls: bool,
-    smtputf8: bool,
 ) -> asyncio.Server:
     logging.info(
-        f"Starting SMTP STARTTLS server {host=}, {port=}, {mails_path=!s}, {bool(ssl_context)=}"
+        f"Starting SMTP STARTTLS server {host=}, {port=}, {mails_path=!s}, {ssl_context != None=}"
     )
     loop = asyncio.get_event_loop()
     return await loop.create_server(
-        partial(
-            protocol_factory_starttls,
-            mails_path,
-            mbox_finder,
-            ssl_context,
-            require_starttls,
-            smtputf8,
-        ),
+        partial(protocol_factory_starttls, mails_path, mbox_finder, ssl_context),
         host=host,
         port=port,
         start_serving=False,
@@ -127,15 +115,14 @@ async def create_smtp_server(
     port: int,
     mails_path: Path,
     mbox_finder: Callable[[str], list[str]],
-    ssl_context: Optional[ssl.SSLContext],
-    smtputf8: bool,
+    ssl_context: Optional[ssl.SSLContext] = None,
 ) -> asyncio.Server:
     logging.info(
-        f"Starting SMTP server {host=}, {port=}, {mails_path=!s}, {bool(ssl_context)=}"
+        f"Starting SMTP server {host=}, {port=}, {mails_path=!s}, {ssl_context != None=}"
     )
     loop = asyncio.get_event_loop()
     return await loop.create_server(
-        partial(protocol_factory, mails_path, mbox_finder, smtputf8),
+        partial(protocol_factory, mails_path, mbox_finder),
         host=host,
         port=port,
         ssl=ssl_context,

tests/test_pop.py

@@ -89,9 +89,6 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
         self.task = asyncio.create_task(pop_server.serve_forever())
         self.reader, self.writer = await asyncio.open_connection("127.0.0.1", 7995)
 
-        # Additional writers to close
-        self.ws: list[asyncio.StreamWriter] = []
-
     async def test_QUIT(self) -> None:
         dialog = """
         S: +OK Server Ready
@@ -136,7 +133,6 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
     async def test_dupe_AUTH(self) -> None:
         r1, w1 = await asyncio.open_connection("127.0.0.1", 7995)
         r2, w2 = await asyncio.open_connection("127.0.0.1", 7995)
-        self.ws += w1, w2
         dialog = """
         S: +OK Server Ready
         C: USER foobar
@@ -235,10 +231,8 @@ class TestPop3(unittest.IsolatedAsyncioTestCase):
 
     async def asyncTearDown(self) -> None:
         logging.debug("at teardown")
-        for w in self.ws + [self.writer]:
-            w.close()
-            await w.wait_closed()
-        self.ws.clear()
+        self.writer.close()
+        await self.writer.wait_closed()
         self.task.cancel("test done")
 
     async def dialog_checker(self, dialog: str) -> None:

tests/test_smtp.py

@@ -33,8 +33,6 @@ class TestSMTP(unittest.IsolatedAsyncioTestCase):
             port=7996,
             mails_path=MAILS_PATH,
             mbox_finder=lambda addr: [TEST_MBOX],
-            ssl_context=None,
-            smtputf8=True,
         )
         self.task = asyncio.create_task(smtp_server.serve_forever())