Coverage for slidge/core/gateway.py: 58%
572 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-11-10 12:26 +0000
1"""
2This module extends slixmpp.ComponentXMPP to make writing new LegacyClients easier
3"""
5from __future__ import annotations
7import abc
8import asyncio
9import logging
10import re
11import sys
12import tempfile
13from collections.abc import (
14 Awaitable,
15 Callable,
16 Coroutine,
17 Iterable,
18 Mapping,
19 Sequence,
20)
21from copy import copy
22from datetime import UTC, datetime, timedelta
23from pathlib import Path
24from typing import (
25 Any,
26 ClassVar,
27 Concatenate,
28 ParamSpec,
29 TypedDict,
30 TypeVar,
31)
33import aiohttp
34from slixmpp import JID, ComponentXMPP, Iq
35from slixmpp.exceptions import IqError, IqTimeout, XMPPError
36from slixmpp.plugins.xep_0004.stanza.form import Form
37from slixmpp.plugins.xep_0060.stanza import OwnerAffiliation
38from slixmpp.plugins.xep_0356.privilege import PrivilegedIqError
39from slixmpp.types import MessageTypes
40from slixmpp.xmlstream.xmlstream import NotConnectedError
41from sqlalchemy.orm import Session as OrmSession
43import slidge.command.categories
44from slidge.command import BUILTIN_COMMANDS
45from slidge.command.adhoc import AdhocProvider
46from slidge.command.admin import Exec
47from slidge.command.base import (
48 Command,
49 CommandBase,
50 ContactCommand,
51 FormField,
52 RoomCommand,
53)
54from slidge.command.chat_command import ChatCommandProvider
55from slidge.command.register import AltRegistrationFlow, Register, RegistrationType
56from slidge.contact import LegacyContact
57from slidge.core import config
58from slidge.core.attachment_upload import AttachmentUploader
59from slidge.core.dispatcher.session_dispatcher import SessionDispatcher
60from slidge.core.mixins.avatar import convert_avatar
61from slidge.core.mixins.message import ContentMessageMixin, InviteMixin
62from slidge.core.pubsub import PubSubComponent
63from slidge.core.session import BaseSession
64from slidge.db import GatewayUser, SlidgeStore
65from slidge.db.avatar import CachedAvatar, avatar_cache
66from slidge.db.models import Attachment
67from slidge.room import LegacyRoom
68from slidge.slixfix.delivery_receipt import DeliveryReceipt
69from slidge.slixfix.roster import RosterBackend
70from slidge.util.jid_escaping import escape_node, unescape_node
71from slidge.util.types import (
72 AnySession,
73 Avatar,
74 LocalpartType,
75 MessageOrPresenceTypeVar,
76 RegistrationValidationCoroutine,
77)
78from slidge.util.util import derive_wired_class
80T = TypeVar("T")
81P = ParamSpec("P")
84class _DefaultRegistrationForm(TypedDict):
85 username: str
86 password: str
89class BaseGateway[
90 SessionT: AnySession = BaseSession,
91 RegistrationFormT: Any = _DefaultRegistrationForm,
92 ModuleDataT: Mapping[str, object] = Any,
93](ComponentXMPP, InviteMixin, ContentMessageMixin, abc.ABC):
94 """
95 The gateway component, handling registrations and un-registrations.
97 On slidge launch, a singleton is instantiated, and it will be made available
98 to public classes such :class:`.LegacyContact` or :class:`.BaseSession` as the
99 ``.xmpp`` attribute.
101 Must be subclassed by a legacy module to set up various aspects of the XMPP
102 component behaviour, such as its display name or welcome message, via
103 class attributes :attr:`.COMPONENT_NAME` :attr:`.WELCOME_MESSAGE`.
105 Abstract methods related to the registration process must be overriden
106 for a functional :term:`Legacy Module`:
108 - :meth:`.validate`
109 - :meth:`.validate_two_factor_code`
110 - :meth:`.get_qr_text`
111 - :meth:`.confirm_qr`
113 NB: Not all of these must be overridden, it depends on the
114 :attr:`REGISTRATION_TYPE`.
116 The other methods, such as :meth:`.send_text` or :meth:`.react` are the same
117 as those of :class:`.LegacyContact` and :class:`.LegacyParticipant`, because
118 the component itself is also a "messaging actor", ie, an :term:`XMPP Entity`.
119 For these methods, you need to specify the JID of the recipient with the
120 `mto` parameter.
122 Since it inherits from :class:`slixmpp.componentxmpp.ComponentXMPP`,you also
123 have a hand on low-level XMPP interactions via slixmpp methods, e.g.:
125 .. code-block:: python
127 self.send_presence(
128 pfrom="somebody@component.example.com",
129 pto="someonwelse@anotherexample.com",
130 )
132 However, you should not need to do so often since the classes of the plugin
133 API provides higher level abstractions around most commonly needed use-cases, such
134 as sending messages, or displaying a custom status.
136 """
138 COMPONENT_NAME: str = NotImplemented
139 """Name of the component, as seen in service discovery by XMPP clients"""
140 COMPONENT_TYPE: str = ""
141 """Type of the gateway, should follow https://xmpp.org/registrar/disco-categories.html"""
142 COMPONENT_AVATAR: Avatar | Path | str | None = None
143 """
144 Path, bytes or URL used by the component as an avatar.
145 """
147 REGISTRATION_FIELDS: ClassVar[Sequence[FormField]] = [
148 FormField(var="username", label="User name", required=True),
149 FormField(var="password", label="Password", required=True, private=True),
150 ]
151 """
152 Iterable of fields presented to the gateway user when registering using :xep:`0077`
153 `extended <https://xmpp.org/extensions/xep-0077.html#extensibility>`_ by :xep:`0004`.
154 """
155 REGISTRATION_INSTRUCTIONS: str = "Enter your credentials"
156 """
157 The text presented to a user who wants to register (or modify) their
158 :term:`legacy <Legacy>` account configuration.
159 """
160 REGISTRATION_TYPE: RegistrationType = RegistrationType.SINGLE_STEP_FORM
161 """
162 This attribute determines how users register to the gateway, ie, how they
163 login to the :term:`legacy network <Legacy Network>`.
164 The credentials are then stored persistently, so this process should happen
165 once per user (unless they unregister).
167 The registration process always start with a basic data form (:xep:`0004`)
168 presented to the user.
169 But the legacy login flow might require something more sophisticated, see
170 :class:`.RegistrationType` for more details.
171 """
173 REGISTRATION_2FA_TITLE = "Enter your 2FA code"
174 REGISTRATION_2FA_INSTRUCTIONS = (
175 "You should have received something via email or SMS, or something"
176 )
177 REGISTRATION_QR_INSTRUCTIONS = "Flash this code or follow this link"
179 ALTERNATIVE_REGISTRATION_FLOWS: ClassVar[
180 list[AltRegistrationFlow[Any, ModuleDataT]]
181 ] = []
183 PREFERENCES: ClassVar[list[FormField]] = [
184 FormField(
185 var="sync_presence",
186 label="Propagate your XMPP presence to the legacy network.",
187 value="true",
188 required=True,
189 type="boolean",
190 ),
191 FormField(
192 var="sync_avatar",
193 label="Propagate your XMPP avatar to the legacy network.",
194 value="true",
195 required=True,
196 type="boolean",
197 ),
198 FormField(
199 var="bookmarks_push",
200 label="Add private rooms to your bookmarks.",
201 value="true",
202 required=True,
203 type="boolean",
204 ),
205 FormField(
206 var="bookmarks_push_public",
207 label="Add public rooms to your bookmarks.",
208 value="false",
209 required=True,
210 type="boolean",
211 ),
212 FormField(
213 var="autojoin_public",
214 label="Auto-join public rooms.",
215 value="false",
216 required=True,
217 type="boolean",
218 ),
219 FormField(
220 var="last_seen_fallback",
221 label="Use contact presence status message to show when they were last seen.",
222 value="true",
223 required=True,
224 type="boolean",
225 ),
226 FormField(
227 var="roster_push",
228 label="Add contacts to your roster.",
229 value="true",
230 required=True,
231 type="boolean",
232 ),
233 FormField(
234 var="reaction_fallback",
235 label="Receive fallback messages for reactions (for legacy XMPP clients)",
236 value="false",
237 required=True,
238 type="boolean",
239 ),
240 ]
242 ROSTER_GROUP: str = "slidge"
243 """
244 Name of the group assigned to a :class:`.LegacyContact` automagically
245 added to the :term:`User`'s roster with :meth:`.LegacyContact.add_to_roster`.
246 """
247 WELCOME_MESSAGE = (
248 "Thank you for registering. Type 'help' to list the available commands, "
249 "or just start messaging away!"
250 )
251 """
252 A welcome message displayed to users on registration.
253 This is useful notably for clients that don't consider component JIDs as a
254 valid recipient in their UI, yet still open a functional chat window on
255 incoming messages from components.
256 """
258 SEARCH_FIELDS: ClassVar[Sequence[FormField]] = [
259 FormField(var="first", label="First name", required=True),
260 FormField(var="last", label="Last name", required=True),
261 FormField(var="phone", label="Phone number", required=False),
262 ]
263 """
264 Fields used for searching items via the component, through :xep:`0055` (jabber search).
265 A common use case is to allow users to search for legacy contacts by something else than
266 their usernames, eg their phone number.
268 Plugins should implement search by overriding :meth:`.BaseSession.search`
269 (restricted to registered users).
271 If there is only one field, it can also be used via the ``jabber:iq:gateway`` protocol
272 described in :xep:`0100`. Limitation: this only works if the search request returns
273 one result item, and if this item has a 'jid' var.
274 """
275 SEARCH_TITLE: str = "Search for legacy contacts"
276 """
277 Title of the search form.
278 """
279 SEARCH_INSTRUCTIONS: str = ""
280 """
281 Instructions of the search form.
282 """
284 MARK_ALL_MESSAGES = False
285 """
286 Set this to True for :term:`legacy networks <Legacy Network>` that expects
287 read marks for *all* messages and not just the latest one that was read
288 (as most XMPP clients will only send a read mark for the latest msg).
289 """
291 PROPER_RECEIPTS = False
292 """
293 Set this to True if the legacy service provides a real equivalent of message delivery receipts
294 (:xep:`0184`), meaning that there is an event thrown when the actual device of a contact receives
295 a message. Make sure to call Contact.received() adequately if this is set to True.
296 """
298 ROOMS = False
299 """
300 This must be set to True if this gateway supports multi-user chats.
301 """
302 SPACES = False
303 """
304 This must be set to True if this gateway supports spaces, cf :xep:`0503`.
305 """
306 THREADS: bool = True
307 """
308 This must be set to True if this gateway supports threads, to enable
309 storing an XMPP/legacy thread ID mapping when necessary.
310 """
312 _mtype: MessageTypes = "chat"
313 is_group = False
314 _can_send_carbon = False
315 store: SlidgeStore
317 _session_cls: type[SessionT]
318 """Concrete :class:`.BaseSession` subclass of this legacy module.
320 Derived automatically from the generic parameter, e.g.,
321 ``class Gateway(BaseGateway[Session])``.
322 """
324 def __init_subclass__(cls, **kwargs: object) -> None:
325 super().__init_subclass__(**kwargs)
326 derive_wired_class(cls, BaseGateway, "_session_cls")
328 COMMANDS: ClassVar[tuple[type[CommandBase], ...]] = BUILTIN_COMMANDS
329 """Commands (adhoc and chatbot) this gateway provides.
331 Subclasses may override this with additional custom commands.
332 E.g.: ``COMMANDS = BaseGateway.COMMANDS + commands_from_module(command)``.
333 """
335 http: aiohttp.ClientSession
336 avatar: CachedAvatar | None = None
338 def __init__(self) -> None:
339 if getattr(self, "_session_cls", None) is None:
340 raise RuntimeError(
341 f"{type(self).__name__}._session_cls is not set. Your gateway"
342 " class must be parameterized with its BaseSession subclass,"
343 " e.g. 'class Gateway(BaseGateway[Session])'."
344 )
345 if config.COMPONENT_NAME:
346 self.COMPONENT_NAME = config.COMPONENT_NAME
347 if config.WELCOME_MESSAGE:
348 self.WELCOME_MESSAGE = config.WELCOME_MESSAGE
349 self.log = log
350 self.datetime_started = datetime.now(tz=UTC)
351 # FIXME: ugly hack to work with the BaseSender mixin :/
352 self.xmpp = self
353 self.default_ns = "jabber:component:accept"
354 super().__init__(
355 config.JID,
356 config.SECRET,
357 config.SERVER,
358 config.PORT,
359 plugin_whitelist=SLIXMPP_PLUGINS,
360 plugin_config={
361 "xep_0077": {
362 "form_fields": None,
363 "form_instructions": self.REGISTRATION_INSTRUCTIONS,
364 "enable_subscription": self.REGISTRATION_TYPE
365 == RegistrationType.SINGLE_STEP_FORM,
366 },
367 "xep_0100": {
368 "component_name": self.COMPONENT_NAME,
369 "type": self.COMPONENT_TYPE,
370 },
371 "xep_0184": {
372 "auto_ack": False,
373 "auto_request": False,
374 },
375 "xep_0363": {
376 "upload_service": config.UPLOAD_SERVICE,
377 },
378 },
379 fix_error_ns=True,
380 )
381 self.loop.set_exception_handler(self.__exception_handler)
382 self.create_task(self.__set_http(), name="set http client")
383 self.has_crashed: bool = False
384 self.use_origin_id = False
386 if config.USER_JID_VALIDATOR is None:
387 config.USER_JID_VALIDATOR = f".*@{self.infer_real_domain()}"
388 log.info(
389 "No USER_JID_VALIDATOR was set, using '%s'.",
390 config.USER_JID_VALIDATOR,
391 )
392 self.jid_validator: re.Pattern[str] = re.compile(config.USER_JID_VALIDATOR)
393 self.qr_pending_registrations = dict[str, asyncio.Future[ModuleDataT | None]]()
395 self.register_plugins()
396 self.__setup_session_cls()
398 self.get_session_from_stanza = self._session_cls.from_stanza
399 self.get_session_from_user = self._session_cls.from_user
401 self.__register_slixmpp_events()
402 self.__register_slixmpp_api()
403 self.roster.set_backend(RosterBackend(self))
405 self.register_plugin("pubsub", {"component_name": self.COMPONENT_NAME})
406 self.pubsub: PubSubComponent = self.plugin["pubsub"] # type:ignore[typeddict-item]
407 self.delivery_receipt = DeliveryReceipt(self)
409 # with this we receive user avatar updates
410 self.plugin["xep_0030"].add_feature("urn:xmpp:avatar:metadata+notify")
412 self.plugin["xep_0030"].add_feature("urn:xmpp:chat-markers:0")
414 if self.ROOMS:
415 self.plugin["xep_0030"].add_feature("http://jabber.org/protocol/muc")
416 self.plugin["xep_0030"].add_feature(self.plugin["xep_0463"].stanza.NS)
417 self.plugin["xep_0030"].add_feature("urn:xmpp:mam:2")
418 self.plugin["xep_0030"].add_feature("urn:xmpp:mam:2#extended")
419 self.plugin["xep_0030"].add_feature(self.plugin["xep_0421"].namespace)
420 self.plugin["xep_0030"].add_feature(self.plugin["xep_0317"].stanza.NS)
421 self.plugin["xep_0030"].add_identity(
422 category="conference",
423 name=self.COMPONENT_NAME,
424 itype="text",
425 jid=self.boundjid,
426 )
427 if self.SPACES:
428 self.plugin["xep_0030"].add_feature("urn:xmpp:spaces:0")
430 self.__adhoc_handler = AdhocProvider(self)
431 self.__chat_commands_handler = ChatCommandProvider(self)
433 self.__dispatcher = SessionDispatcher[RegistrationFormT](self)
434 self.get_caps_ver = self.__dispatcher.get_caps_ver
435 self._populate_caps_cache = self.__dispatcher.populate_cache
437 self.__register_commands()
439 def __setup_session_cls(self) -> None:
440 contact_cls = self._session_cls._roster_cls._contact_cls
442 if contact_cls.REACTIONS_SINGLE_EMOJI:
443 form = Form()
444 form["type"] = "result"
445 form.add_field(
446 "FORM_TYPE", "hidden", value="urn:xmpp:reactions:0:restrictions"
447 )
448 form.add_field("max_reactions_per_user", value="1", type="text-single")
449 form.add_field("scope", value="domain")
450 self.plugin["xep_0128"].add_extended_info(data=form)
452 self._session_cls.xmpp = self
454 @property
455 def _uploader(self) -> AttachmentUploader:
456 # the component itself is not bound to any user session
457 return AttachmentUploader(self, None)
459 async def kill_session(self, jid: JID) -> None:
460 await self._session_cls.kill_by_jid(jid)
462 async def __set_http(self) -> None:
463 self.http = aiohttp.ClientSession()
464 if getattr(self, "_test_mode", False):
465 return
466 avatar_cache.http = self.http
468 def __register_commands(self) -> None:
469 for cls in self.COMMANDS:
470 if any(x is NotImplemented for x in [cls.CHAT_COMMAND, cls.NODE, cls.NAME]):
471 raise TypeError(
472 f"{cls} is listed in {type(self).__name__}.COMMANDS but"
473 " does not set NAME, NODE and CHAT_COMMAND."
474 )
475 if issubclass(cls, ContactCommand):
476 LegacyContact._commands[cls.NODE] = cls
477 LegacyContact._commands_chat[cls.CHAT_COMMAND] = cls
478 continue
479 if issubclass(cls, RoomCommand):
480 LegacyRoom._commands[cls.NODE] = cls
481 LegacyRoom._commands_chat[cls.CHAT_COMMAND] = cls
482 continue
483 if not issubclass(cls, Command):
484 raise TypeError(
485 f"{cls} is listed in {type(self).__name__}.COMMANDS but is"
486 " not a Command, ContactCommand or RoomCommand subclass."
487 )
488 if cls is Exec:
489 if config.DEV_MODE:
490 log.warning(r"/!\ DEV MODE ENABLED /!\\")
491 else:
492 continue
493 if cls.CATEGORY == slidge.command.categories.ROOMS and not self.ROOMS:
494 continue
495 if cls.CATEGORY == slidge.command.categories.SPACES and not self.SPACES:
496 continue
497 c = cls(self)
498 log.debug("Registering %s", cls)
499 self.__adhoc_handler.register(c)
500 self.__chat_commands_handler.register(c)
502 for i, alt in enumerate(self.ALTERNATIVE_REGISTRATION_FLOWS):
504 class AltRegister(Register[RegistrationFormT, ModuleDataT]):
505 NAME = alt.title
506 NODE = f"https://slidge.im/register/alt{i}"
507 CHAT_COMMAND = f"register-{i}"
508 _instructions = alt.instructions
509 _type = alt.type
510 _fields = alt.fields
511 _validate = staticmethod(alt.validate) if alt.validate else None
513 inst = AltRegister(self)
514 log.debug("Registering alternative registration flow: %s", alt.title)
515 self.__adhoc_handler.register(inst)
516 self.__chat_commands_handler.register(inst)
518 def __exception_handler(
519 self, loop: asyncio.AbstractEventLoop, context: dict[Any, Any]
520 ) -> None:
521 """
522 Called when the event loop encounters an exception that was not
523 handled anywhere else.
525 Tasks created via :meth:`create_task`, are handled separately and do not reach
526 this handler.
527 """
528 log.debug("Context in the exception handler: %s", context)
529 exc = context.get("exception")
530 if exc is None:
531 log.debug("No exception in this context: %s", context)
532 elif isinstance(exc, SystemExit):
533 log.debug("SystemExit called in an asyncio task")
534 elif context.get("future") is not None:
535 log.warning(
536 "Exception in a third-party task was never retrieved: %s",
537 context,
538 exc_info=exc,
539 )
540 else:
541 log.error("Crash in an asyncio task: %s", context)
542 log.exception("Crash in task", exc_info=exc)
543 self.has_crashed = True
544 loop.stop()
546 def create_task(
547 self,
548 coro: Coroutine[Any, Any, Any],
549 *,
550 name: str | None = None,
551 ) -> asyncio.Task[Any]:
552 """Create a task that crashes the gateway if it fails."""
553 task = self.loop.create_task(coro, name=name)
554 task.add_done_callback(self.__handle_task_result)
555 return task
557 def __handle_task_result(self, task: asyncio.Task[Any]) -> None:
558 if task.cancelled():
559 return
560 exc = task.exception()
561 if exc is not None:
562 log.error(
563 "Crash in task %s",
564 task.get_name(),
565 exc_info=(type(exc), exc, exc.__traceback__),
566 )
567 self.has_crashed = True
568 self.loop.stop()
570 def __register_slixmpp_events(self) -> None:
571 self.del_event_handler("presence_subscribe", self._handle_subscribe)
572 self.del_event_handler("presence_unsubscribe", self._handle_unsubscribe)
573 self.del_event_handler("presence_subscribed", self._handle_subscribed)
574 self.del_event_handler("presence_unsubscribed", self._handle_unsubscribed)
575 self.del_event_handler(
576 "roster_subscription_request", self._handle_new_subscription
577 )
578 self.del_event_handler("presence_probe", self._handle_probe)
579 self.add_event_handler("session_start", self.__on_session_start)
581 def __register_slixmpp_api(self) -> None:
582 def with_session(
583 func: Callable[Concatenate[OrmSession, P], T], commit: bool = True
584 ) -> Callable[P, T]:
585 def wrapped(*a: P.args, **kw: P.kwargs) -> T:
586 with self.store.session() as orm:
587 res = func(orm, *a, **kw)
588 if commit:
589 orm.commit()
590 return res
592 return wrapped
594 self.plugin["xep_0231"].api.register(
595 with_session(self.store.bob.get_bob, False), "get_bob"
596 )
597 self.plugin["xep_0231"].api.register(
598 with_session(self.store.bob.set_bob), "set_bob"
599 )
600 self.plugin["xep_0231"].api.register(
601 with_session(self.store.bob.del_bob), "del_bob"
602 )
604 @property # type:ignore[override]
605 def jid(self) -> JID:
606 # Override to avoid slixmpp deprecation warnings.
607 return self.boundjid
609 @jid.setter
610 def jid(self, jid: JID) -> None:
611 raise RuntimeError
613 async def __on_session_start(self, event: object) -> None:
614 log.debug("Gateway session start: %s", event)
616 await self.__setup_attachments()
618 # prevents XMPP clients from considering the gateway as an HTTP upload
619 disco = self.plugin["xep_0030"]
620 await disco.del_feature(feature="urn:xmpp:http:upload:0", jid=self.boundjid)
621 await self.plugin["xep_0115"].update_caps(jid=self.boundjid)
623 if self.COMPONENT_AVATAR is not None:
624 log.debug("Setting gateway avatar…")
625 avatar = convert_avatar(self.COMPONENT_AVATAR, "!!---slidge---special---")
626 assert avatar is not None
627 try:
628 cached_avatar = await avatar_cache.get(avatar)
629 except Exception as e:
630 log.exception("Could not set the component avatar.", exc_info=e)
631 cached_avatar = None
632 else:
633 assert cached_avatar is not None
634 self.avatar = cached_avatar
635 else:
636 cached_avatar = None
638 with self.store.session() as orm:
639 users = orm.query(GatewayUser).all()
640 for user in users:
641 # TODO: before this, we should check if the user has removed us from their roster
642 # while we were offline and trigger unregister from there. Presence probe does not seem
643 # to work in this case, there must be another way. privileged entity could be used
644 # as last resort.
645 try:
646 await self["xep_0100"].add_component_to_roster(user.jid)
647 await self.__add_component_to_mds_whitelist(user.jid)
648 except (IqError, IqTimeout) as e:
649 # TODO: remove the user when this happens? or at least
650 # this can happen when the user has unsubscribed from the XMPP server
651 log.warning(
652 "Error with user %s, not logging them automatically",
653 user,
654 exc_info=e,
655 )
656 continue
657 session = self._session_cls.from_user(user)
658 session.create_task(self.login_wrap(session), name=f"login wrap of {user}")
659 if cached_avatar is not None:
660 await self.pubsub.broadcast_avatar(
661 self.boundjid.bare, session.user_jid, cached_avatar
662 )
664 log.info("Slidge has successfully started")
666 async def __setup_attachments(self) -> None:
667 expire_coro = self.__expire_attachments_upload
668 if config.NO_UPLOAD_PATH:
669 if config.NO_UPLOAD_URL_PREFIX is None:
670 raise RuntimeError(
671 "If you set NO_UPLOAD_PATH you must set NO_UPLOAD_URL_PREFIX too."
672 )
673 expire_coro = self.__expire_attachments_no_upload
674 elif not config.UPLOAD_SERVICE:
675 try:
676 info_iq = await self.xmpp.plugin["xep_0363"].find_upload_service(
677 self.infer_real_domain()
678 )
679 except XMPPError:
680 info_iq = None
681 log.exception(
682 "The upload service could not be automatically determine. "
683 "Attachments to XMPP will not work. "
684 "Either specify 'upload-service' or 'no-upload-path' to fix that."
685 )
686 if info_iq is None:
687 if self.REGISTRATION_TYPE == RegistrationType.QRCODE:
688 log.warning(
689 "No method was configured for attachment and slidge "
690 "could not automatically determine the JID of a usable upload service. "
691 "Users likely won't be able to register since this network uses a "
692 "QR-code based registration flow."
693 )
694 if not config.USE_ATTACHMENT_ORIGINAL_URLS:
695 log.warning(
696 "Setting USE_ATTACHMENT_ORIGINAL_URLS to True since no method was configured "
697 "for attachments and no upload service was found. NB: this does not work for all "
698 "networks, especially for the E2EE attachments."
699 )
700 config.USE_ATTACHMENT_ORIGINAL_URLS = True
701 else:
702 log.info("Auto-discovered upload service: %s", info_iq["from"])
703 config.UPLOAD_SERVICE = info_iq["from"]
705 self.__expire_attachments_task = self.create_task(_loop(expire_coro, 3600 * 24))
707 async def __expire_attachments_no_upload(self) -> None:
708 with self.store.session() as orm:
709 attachments = self.store.attachments.get_all(orm)
710 self.__expire_attachments_in_store(
711 await asyncio.to_thread(self.__rm_local_attachments, attachments)
712 )
714 def __rm_local_attachments(self, attachments: list[Attachment]) -> list[int]:
715 to_remove = []
716 cutoff = datetime.now(tz=UTC) - timedelta(
717 days=config.NO_UPLOAD_MAX_DAYS or config.MAM_MAX_DAYS
718 )
720 for attachment in attachments:
721 path = attachment.local_path
722 try:
723 created = datetime.fromtimestamp(path.stat().st_ctime, tz=UTC)
724 except OSError:
725 self.log.debug(
726 "mtime of %s could not be determined, clearing up the row",
727 attachment,
728 )
729 to_remove.append(attachment.id)
730 else:
731 if created > cutoff:
732 continue
733 to_remove.append(attachment.id)
734 self.log.debug("%s is too old", attachment)
735 _safe_rm_parent(path)
736 to_remove.append(attachment.id)
738 return to_remove
740 async def __expire_attachments_upload(self) -> None:
741 with self.store.session() as orm:
742 attachments = self.store.attachments.get_all(orm)
743 to_remove = []
744 for attachment in attachments:
745 try:
746 async with self.http.head(attachment.url) as resp:
747 if not resp.ok:
748 self.log.debug("%s is not reachable anymore", attachment)
749 to_remove.append(attachment.id)
750 except Exception as e:
751 self.log.debug("Error when trying to reach %s", attachment, exc_info=e)
752 to_remove.append(attachment.id)
753 self.__expire_attachments_in_store(to_remove)
755 def __expire_attachments_in_store(self, to_remove: list[int]) -> None:
756 with self.store.session() as orm:
757 self.store.attachments.remove(orm, to_remove)
758 orm.commit()
760 def infer_real_domain(self) -> JID:
761 return JID(re.sub(r"^.*?\.", "", self.xmpp.boundjid.bare))
763 async def __add_component_to_mds_whitelist(self, user_jid: JID) -> None:
764 # Uses privileged entity to add ourselves to the whitelist of the PEP
765 # MDS node so we receive MDS events
766 iq_creation = Iq(sto=user_jid.bare, sfrom=user_jid, stype="set")
767 iq_creation["pubsub"]["create"]["node"] = self.plugin["xep_0490"].stanza.NS
769 try:
770 await self.plugin["xep_0356"].send_privileged_iq(iq_creation)
771 except PermissionError:
772 log.warning(
773 "IQ privileges not granted for pubsub namespace, we cannot "
774 "create the MDS node of %s",
775 user_jid,
776 )
777 except PrivilegedIqError as exc:
778 nested = exc.nested_error()
779 # conflict this means the node already exists, we can ignore that
780 if nested is not None and nested.condition != "conflict":
781 log.exception(
782 "Could not create the MDS node of %s", user_jid, exc_info=exc
783 )
784 except Exception as e:
785 log.exception(
786 "Error while trying to create to the MDS node of %s",
787 user_jid,
788 exc_info=e,
789 )
791 iq_affiliation = Iq(sto=user_jid.bare, sfrom=user_jid, stype="set")
792 iq_affiliation["pubsub_owner"]["affiliations"]["node"] = self.plugin[
793 "xep_0490"
794 ].stanza.NS
796 aff = OwnerAffiliation()
797 aff["jid"] = self.boundjid.bare
798 aff["affiliation"] = "member"
799 iq_affiliation["pubsub_owner"]["affiliations"].append(aff)
801 try:
802 await self.plugin["xep_0356"].send_privileged_iq(iq_affiliation)
803 except PermissionError:
804 log.warning(
805 "IQ privileges not granted for pubsub#owner namespace, we cannot "
806 "listen to the MDS events of %s",
807 user_jid,
808 )
809 except Exception as e:
810 log.exception(
811 "Error while trying to subscribe to the MDS node of %s",
812 user_jid,
813 exc_info=e,
814 )
816 async def login_wrap(self, session: AnySession) -> str:
817 session.log.debug("In login_wrap…")
818 session.send_gateway_status("Logging in…", show="dnd")
819 session.is_logging_in = True
820 try:
821 status = await session.login()
822 except Exception as e:
823 log.warning("Login problem for %s", session.user_jid, exc_info=e)
824 session.send_gateway_status(f"Could not login: {e}", show="dnd")
825 msg = (
826 "You are not connected to this gateway! "
827 f"Maybe this message will tell you why: {e}"
828 )
829 session.send_gateway_message(msg)
830 session.logged = False
831 session.send_gateway_status("Login failed", show="dnd")
832 return msg
834 session.log.info("Login success")
835 session.logged = True
836 session.log.debug("Sync contacts…")
837 session.send_gateway_status("Syncing contacts…", show="dnd")
838 await session.contacts._fill()
839 if not (r := session.contacts.ready).done():
840 r.set_result(True)
841 if self.ROOMS:
842 session.log.debug("Sync rooms…")
843 session.send_gateway_status("Syncing rooms…", show="dnd")
844 await session.rooms._fill()
845 if not (r := session.rooms.ready).done():
846 r.set_result(True)
847 session.log.debug("Probe")
848 self.send_presence(pto=session.user.jid.bare, ptype="probe")
849 if status is None:
850 status = "Logged in"
851 session.send_gateway_status(status, show="chat")
853 if session.preferences.get("sync_avatar", False):
854 session.create_task(
855 self.fetch_user_avatar(session), name=f"fetch avatar of {session.user}"
856 )
857 else:
858 with self.store.session(expire_on_commit=False) as orm:
859 session.user.avatar_hash = None
860 orm.add(session.user)
861 orm.commit()
862 return status
864 async def fetch_user_avatar(self, session: AnySession) -> None:
865 try:
866 iq = await self.xmpp.plugin["xep_0060"].get_items(
867 session.user_jid.bare,
868 self.xmpp.plugin["xep_0084"].stanza.MetaData.namespace,
869 ifrom=self.boundjid.bare,
870 )
871 except IqTimeout:
872 self.log.warning("Iq timeout trying to fetch user avatar")
873 return
874 except IqError as e:
875 self.log.debug("Iq error when trying to fetch user avatar: %s", e)
876 if e.condition == "item-not-found":
877 try:
878 await session.on_avatar(None, None, None, None, None)
879 except NotImplementedError:
880 pass
881 else:
882 with self.store.session(expire_on_commit=False) as orm:
883 session.user.avatar_hash = None
884 orm.add(session.user)
885 orm.commit()
886 return
887 await self.__dispatcher.on_avatar_metadata_info(
888 session, iq["pubsub"]["items"]["item"]["avatar_metadata"]["info"]
889 )
891 def _send(
892 self,
893 stanza: MessageOrPresenceTypeVar,
894 **send_kwargs: Any, # noqa:ANN401
895 ) -> MessageOrPresenceTypeVar:
896 stanza.set_from(self.boundjid.bare)
897 if mto := send_kwargs.get("mto"):
898 stanza.set_to(mto)
899 stanza.send()
900 return stanza
902 def raise_if_not_allowed_jid(self, jid: JID) -> None:
903 if not self.jid_validator.match(jid.bare):
904 raise XMPPError(
905 condition="not-allowed",
906 text="Your account is not allowed to use this gateway. "
907 "The admin controls that with the USER_JID_VALIDATOR option.",
908 )
910 def send_raw(self, data: str | bytes) -> None:
911 # overridden from XMLStream to strip base64-encoded data from the logs
912 # to make them more readable.
913 if log.isEnabledFor(level=logging.DEBUG):
914 stripped = copy(data) if isinstance(data, str) else data.decode("utf-8")
915 # there is probably a way to do that in a single RE,
916 # but since it's only for debugging, the perf penalty
917 # does not matter much
918 for el in LOG_STRIP_ELEMENTS:
919 stripped = re.sub(
920 f"(<{el}.*?>)(.*)(</{el}>)",
921 "\1[STRIPPED]\3",
922 stripped,
923 flags=re.DOTALL | re.IGNORECASE,
924 )
925 log.debug("SEND: %s", stripped)
926 if not self.transport:
927 raise NotConnectedError()
928 if isinstance(data, str):
929 data = data.encode("utf-8")
930 self.transport.write(data)
932 def get_session_from_jid(self, j: JID) -> BaseSession | None:
933 try:
934 return self._session_cls.from_jid(j)
935 except XMPPError:
936 return None
938 def exception(self, exception: Exception) -> None:
939 # """
940 # Called when a task created by slixmpp's internal (eg, on slix events) raises an Exception.
941 #
942 # Stop the event loop and exit on unhandled exception.
943 #
944 # The default :class:`slixmpp.basexmpp.BaseXMPP` behaviour is just to
945 # log the exception, but we want to avoid undefined behaviour.
946 #
947 # :param exception: An unhandled :class:`Exception` object.
948 # """
949 if isinstance(exception, IqError):
950 iq = exception.iq
951 log.error("%s: %s", iq["error"]["condition"], iq["error"]["text"])
952 log.warning("You should catch IqError exceptions")
953 elif isinstance(exception, IqTimeout):
954 iq = exception.iq
955 log.error("Request timed out: %s", iq)
956 log.warning("You should catch IqTimeout exceptions")
957 elif isinstance(exception, SyntaxError):
958 # Hide stream parsing errors that occur when the
959 # stream is disconnected (they've been handled, we
960 # don't need to make a mess in the logs).
961 pass
962 else:
963 if exception:
964 log.exception(exception)
965 self.loop.stop()
966 sys.exit(1)
968 async def make_registration_form(
969 self, _jid: JID, _node: str, _ifrom: JID, iq: Iq
970 ) -> Iq:
971 self.raise_if_not_allowed_jid(iq.get_from())
972 reg = iq["register"]
973 with self.store.session() as orm:
974 user = (
975 orm.query(GatewayUser).filter_by(jid=iq.get_from().bare).one_or_none()
976 )
977 log.debug("User found: %s", user)
979 form = reg["form"]
980 form.add_field(
981 "FORM_TYPE",
982 ftype="hidden",
983 value="jabber:iq:register",
984 )
985 form["title"] = f"Registration to '{self.COMPONENT_NAME}'"
986 form["instructions"] = self.REGISTRATION_INSTRUCTIONS
988 if user is not None:
989 reg["registered"] = False
990 form.add_field(
991 "remove",
992 label="Remove my registration",
993 required=True,
994 ftype="boolean",
995 value=False,
996 )
998 for field in self.REGISTRATION_FIELDS:
999 if field.var in reg.interfaces:
1000 val = None if user is None else user.get(field.var)
1001 if val is None:
1002 reg.add_field(field.var)
1003 else:
1004 reg[field.var] = val
1006 reg["instructions"] = self.REGISTRATION_INSTRUCTIONS
1008 for field in self.REGISTRATION_FIELDS:
1009 form.add_field(
1010 field.var,
1011 label=field.label,
1012 required=field.required,
1013 ftype=field.type,
1014 options=field.options,
1015 value=field.value if user is None else user.get(field.var, field.value),
1016 )
1018 reply = iq.reply()
1019 reply.set_payload(reg)
1020 return reply # type:ignore[no-any-return]
1022 async def user_prevalidate(
1023 self,
1024 ifrom: JID,
1025 form_dict: RegistrationFormT,
1026 fields: Iterable[FormField] | None = None,
1027 validate: RegistrationValidationCoroutine[RegistrationFormT, ModuleDataT]
1028 | None = None,
1029 ) -> ModuleDataT | None:
1030 # Pre validate a registration form using the content of self.REGISTRATION_FIELDS
1031 # before passing it to the plugin custom validation logic
1032 if fields is None:
1033 fields = self.REGISTRATION_FIELDS
1034 for field in fields:
1035 if field.required and not form_dict.get(field.var):
1036 raise ValueError(f"Missing field: '{field.label}'")
1037 if validate:
1038 return await validate(ifrom, form_dict)
1039 else:
1040 return await self.validate(ifrom, form_dict)
1042 @abc.abstractmethod
1043 async def validate(
1044 self, user_jid: JID, registration_form: RegistrationFormT
1045 ) -> ModuleDataT | None:
1046 """
1047 Validate a user's initial registration form.
1049 Should raise the appropriate :class:`slixmpp.exceptions.XMPPError`
1050 if the registration does not allow to continue the registration process.
1052 If :py:attr:`REGISTRATION_TYPE` is a
1053 :attr:`.RegistrationType.SINGLE_STEP_FORM`,
1054 this method should raise something if it wasn't possible to successfully
1055 log in to the legacy service with the registration form content.
1057 It is also used for other types of :py:attr:`REGISTRATION_TYPE` too, since
1058 the first step is always a form. If :attr:`.REGISTRATION_FIELDS` is an
1059 empty list (ie, it declares no :class:`.FormField`), the "form" is
1060 effectively a confirmation dialog displaying
1061 :attr:`.REGISTRATION_INSTRUCTIONS`.
1063 :param user_jid: JID of the user that has just registered
1064 :param registration_form: A dict where keys are the :attr:`.FormField.var` attributes
1065 of the :attr:`.BaseGateway.REGISTRATION_FIELDS` iterable.
1066 This dict can be modified and will be accessible as the ``legacy_module_data``
1067 of the
1069 :return : A dict that will be stored as the persistent "legacy_module_data"
1070 for this user. If you don't return anything here, the whole registration_form
1071 content will be stored.
1072 """
1073 raise NotImplementedError
1075 async def validate_two_factor_code(
1076 self, user: GatewayUser, code: str
1077 ) -> ModuleDataT | None:
1078 """
1079 Called when the user enters their 2FA code.
1081 Should raise the appropriate :class:`slixmpp.exceptions.XMPPError`
1082 if the login fails, and return successfully otherwise.
1084 Only used when :attr:`REGISTRATION_TYPE` is
1085 :attr:`.RegistrationType.TWO_FACTOR_CODE`.
1087 :param user: The :class:`.GatewayUser` whose registration is pending
1088 Use their :attr:`.GatewayUser.bare_jid` and/or
1089 :attr:`.registration_form` attributes to get what you need.
1090 :param code: The code they entered, either via "chatbot" message or
1091 adhoc command
1093 :return : A dict which keys and values will be added to the persistent "legacy_module_data"
1094 for this user.
1095 """
1096 raise NotImplementedError
1098 async def get_qr_text(self, user: GatewayUser) -> str:
1099 """
1100 This is where slidge gets the QR code content for the QR-based
1101 registration process. It will turn it into a QR code image and send it
1102 to the not-yet-fully-registered :class:`.GatewayUser`.
1104 Only used in when :attr:`BaseGateway.REGISTRATION_TYPE` is
1105 :attr:`.RegistrationType.QRCODE`.
1107 :param user: The :class:`.GatewayUser` whose registration is pending
1108 Use their :attr:`.GatewayUser.bare_jid` and/or
1109 :attr:`.registration_form` attributes to get what you need.
1110 """
1111 raise NotImplementedError
1113 async def confirm_qr(
1114 self,
1115 user_bare_jid: str,
1116 exception: Exception | None = None,
1117 legacy_data: ModuleDataT | None = None,
1118 ) -> None:
1119 """
1120 This method is meant to be called to finalize QR code-based registration
1121 flows, once the legacy service confirms the QR flashing.
1123 Only used in when :attr:`BaseGateway.REGISTRATION_TYPE` is
1124 :attr:`.RegistrationType.QRCODE`.
1126 :param user_bare_jid: The bare JID of the almost-registered
1127 :class:`GatewayUser` instance
1128 :param exception: Optionally, an XMPPError to be raised to **not** confirm
1129 QR code flashing.
1130 :param legacy_data: dict which keys and values will be added to the persistent
1131 "legacy_module_data" for this user.
1132 """
1133 fut = self.qr_pending_registrations[user_bare_jid]
1134 if exception is None:
1135 fut.set_result(legacy_data)
1136 else:
1137 fut.set_exception(exception)
1139 async def unregister_user(
1140 self, user: GatewayUser, msg: str = "You unregistered from this gateway."
1141 ) -> None:
1142 self.send_presence(pshow="dnd", pstatus=msg, pto=user.jid)
1143 await self.xmpp.plugin["xep_0077"].api["user_remove"](None, None, user.jid) # type:ignore[call-arg]
1144 await self.xmpp._session_cls.kill_by_jid(user.jid)
1146 async def input(
1147 self,
1148 jid: JID,
1149 text: str | None = None,
1150 mtype: MessageTypes = "chat",
1151 **input_kwargs: Any, # noqa:ANN401
1152 ) -> str:
1153 """
1154 Request arbitrary user input using a simple chat message, and await the result.
1156 You shouldn't need to call this directly bust instead use
1157 :meth:`.BaseSession.input` to directly target a user.
1159 :param jid: The JID we want input from
1160 :param text: A prompt to display for the user
1161 :param mtype: Message type
1162 :return: The user's reply
1163 """
1164 return await self.__chat_commands_handler.input(
1165 jid, text, mtype=mtype, **input_kwargs
1166 )
1168 async def send_qr(
1169 self,
1170 text: str,
1171 **msg_kwargs: Any, # noqa:ANN401
1172 ) -> str | None:
1173 """
1174 Sends a QR Code to a JID
1176 You shouldn't need to call directly bust instead use
1177 :meth:`.BaseSession.send_qr` to directly target a user.
1179 :param text: The text that will be converted to a QR Code
1180 :param msg_kwargs: Optional additional arguments to pass to
1181 :meth:`.BaseGateway.send_file`, such as the recipient of the QR,
1182 code
1183 """
1184 try:
1185 import qrcode
1186 except ImportError:
1187 log.error("Slidge needs the [qr] extra to be able to generate QR codes")
1188 raise
1189 qr = qrcode.make(text)
1190 with tempfile.NamedTemporaryFile(suffix=".png") as f:
1191 qr.save(f.name)
1192 url, _msgs = await self.send_file(Path(f.name), **msg_kwargs)
1193 return url
1195 def shutdown(self) -> list[asyncio.Task[None]]:
1196 # """
1197 # Called by the slidge entrypoint on normal exit.
1198 #
1199 # Sends offline presences from all contacts of all user sessions and from
1200 # the gateway component itself.
1201 # No need to call this manually, :func:`slidge.__main__.main` should take care of it.
1202 # """
1203 log.debug("Shutting down")
1204 tasks = []
1205 with self.store.session() as orm:
1206 for user in orm.query(GatewayUser).all():
1207 tasks.append(self._session_cls.from_jid(user.jid).shutdown())
1208 self.send_presence(ptype="unavailable", pto=user.jid)
1209 return tasks
1211 def localpart_to_legacy_id(self, localpart: str) -> tuple[LocalpartType, str]:
1212 """
1213 Converts a JID local part to a legacy ID, specifying whether this is a
1214 group or a contact.
1216 The default implementation mandates the "room-" and "contact-"
1217 prefixes and applies JID (un)escaping (:xep:`0106`).
1219 Should be overridden if you want to customize how legacy IDs map
1220 to JIDs. This must be overridden if the default conversion is not
1221 enough, e.g., if legacy IDs are case-sensitive.
1223 No network calls are expected, but if a legacy ID is not of the right
1224 type (e.g., it contains letters when it should only be numbers), this
1225 should raise an ``XMPPError``.
1227 Implementations must ensure consistency with
1228 :meth:`.legacy_id_to_localpart`.
1230 :param localpart: The local part of a JID (aka "user part" or
1231 sometimes "node part").
1232 :return: (legacy entity type, legacy ID).
1233 """
1234 if localpart.startswith("room-"):
1235 return "room", unescape_node(localpart.removeprefix("room-"))
1236 elif localpart.startswith("contact-"):
1237 return "contact", unescape_node(localpart.removeprefix("contact-"))
1238 else:
1239 raise XMPPError("bad-request")
1241 def legacy_id_to_localpart(self, type_: LocalpartType, legacy_id: str) -> str:
1242 """
1243 Converts a legacy ID into a JID local part.
1245 The default implementation adds the "room-" and "contact-"
1246 prefixes and applies JID escaping (:xep:`0106`).
1248 Should be overridden if you want to customize how legacy IDs map to
1249 JIDs. This must be overridden if the default conversion is not enough,
1250 e.g., if legacy IDs are case-sensitive.
1252 Implementations must ensure consistency with
1253 :meth:`.localpart_to_legacy_id`.
1255 :param type_: Whether this is a "contact" or "room" legacy ID.
1256 :param legacy_id: The legacy ID.
1258 :return: A valid JID local part.
1259 """
1260 match type_:
1261 case "room":
1262 return f"room-{escape_node(legacy_id)}"
1263 case "contact":
1264 return f"contact-{escape_node(legacy_id)}"
1265 case _:
1266 raise XMPPError("bad-request")
1269SLIXMPP_PLUGINS = [
1270 "xep_0030", # Service discovery
1271 "xep_0045", # Multi-User Chat
1272 "xep_0050", # Adhoc commands
1273 "xep_0054", # VCard-temp (for MUC avatars)
1274 "xep_0055", # Jabber search
1275 "xep_0059", # Result Set Management
1276 "xep_0066", # Out of Band Data
1277 "xep_0071", # XHTML-IM (for stickers and custom emojis maybe later)
1278 "xep_0077", # In-band registration
1279 "xep_0084", # User Avatar
1280 "xep_0085", # Chat state notifications
1281 "xep_0100", # Gateway interaction
1282 "xep_0106", # JID Escaping
1283 "xep_0115", # Entity capabilities
1284 "xep_0122", # Data Forms Validation
1285 "xep_0128", # Service Discovery Extensions
1286 "xep_0153", # vCard-Based Avatars (for MUC avatars)
1287 "xep_0172", # User nickname
1288 "xep_0184", # Message Delivery Receipts
1289 "xep_0199", # XMPP Ping
1290 "xep_0221", # Data Forms Media Element
1291 "xep_0231", # Bits of Binary (for stickers and custom emojis maybe later)
1292 "xep_0249", # Direct MUC Invitations
1293 "xep_0264", # Jingle Content Thumbnails
1294 "xep_0280", # Carbons
1295 "xep_0292_provider", # VCard4
1296 "xep_0308", # Last message correction
1297 "xep_0313", # Message Archive Management
1298 "xep_0317", # Hats
1299 "xep_0319", # Last User Interaction in Presence
1300 "xep_0333", # Chat markers
1301 "xep_0334", # Message Processing Hints
1302 "xep_0356", # Privileged Entity
1303 "xep_0363", # HTTP file upload
1304 "xep_0385", # Stateless in-line media sharing
1305 "xep_0402", # PEP Native Bookmarks
1306 "xep_0421", # Anonymous unique occupant identifiers for MUCs
1307 "xep_0424", # Message retraction
1308 "xep_0425", # Message moderation
1309 "xep_0444", # Message reactions
1310 "xep_0447", # Stateless File Sharing
1311 "xep_0449", # Stickers
1312 "xep_0461", # Message replies
1313 "xep_0462", # Pubsub Type Filtering
1314 "xep_0463", # MUC Affiliation Versioning
1315 "xep_0469", # Bookmark Pinning
1316 "xep_0490", # Message Displayed Synchronization
1317 "xep_0492", # Chat Notification Settings
1318 # "xep_0503", # Server-side spaces
1319 "xep_0511", # Link Metadata
1320]
1323async def _loop(coro: Callable[[], Awaitable[None]], sleep: int) -> None:
1324 while True:
1325 await coro()
1326 await asyncio.sleep(sleep)
1329def _safe_rm_parent(path: Path) -> None:
1330 try:
1331 path.unlink()
1332 path.parent.rmdir()
1333 except OSError as exc:
1334 log.warning("%s couldn't be cleanly removed: !r", exc)
1337LOG_STRIP_ELEMENTS = ["data", "binval"]
1339log = logging.getLogger(__name__)