Add SNS Action for Spidermon Notifications (#413)

* feat: add sns directory and skeleton code

* feat: add notification types to sns

* fix: format

* fix: format precommit

* fix: format

* docs: add docs for sns

* fix: rm comments

* fix: ident

* fix: fmt

---------

Co-authored-by: felipdc <felipe@HP-LaptopFelipe>

docs/source/actions/index.rst

@@ -18,4 +18,5 @@ You can define your own actions or use one of the existing built-in actions.
    job-tags-action
    file-report-action
    sentry-action
+   sns-action
    custom-action

docs/source/actions/sns-action.rst

@@ -0,0 +1,52 @@
+SNS action
+==========
+
+This action allows you to send custom notifications to an AWS `Simple Notification Service (SNS)`_ topic when your monitor suites finish their execution.
+
+To use this action, you need to provide the AWS credentials and the SNS topic ARN in your ``settings.py`` file as follows:
+
+.. code-block:: python
+
+    # settings.py
+    SPIDERMON_SNS_TOPIC_ARN = '<SNS_TOPIC_ARN>'
+    SPIDERMON_AWS_ACCESS_KEY_ID = '<AWS_ACCESS_KEY>'
+    SPIDERMON_AWS_SECRET_ACCESS_KEY = '<AWS_SECRET_KEY>'
+    SPIDERMON_AWS_REGION_NAME = '<AWS_REGION_NAME>'  # Default is 'us-east-1'
+
+A notification sent to the SNS topic can be further integrated with other AWS services or third-party applications.
+
+The following settings are the minimum needed to make this action work:
+
+SPIDERMON_SNS_TOPIC_ARN
+-----------------------
+
+ARN (Amazon Resource Name) of the SNS topic where the message will be published.
+
+SPIDERMON_AWS_ACCESS_KEY_ID
+---------------------------
+
+Your AWS access key ID.
+
+SPIDERMON_AWS_SECRET_ACCESS_KEY
+-------------------------------
+
+Your AWS secret access key.
+
+.. warning::
+
+    Be careful when using AWS credentials in Spidermon. Do not publish your AWS access key ID or secret access key in public code repositories.
+
+Other settings available:
+
+SPIDERMON_AWS_REGION_NAME
+-------------------------
+
+Default: ``us-east-1``
+
+The AWS region where your SNS topic is located.
+
+.. note::
+
+    Ensure that the AWS user associated with the provided credentials has the necessary permissions to publish messages to the specified SNS topic.
+
+.. _`Simple Notification Service (SNS)`: https://aws.amazon.com/sns/

spidermon/contrib/actions/sns/__init__.py

@@ -0,0 +1,73 @@
+import boto3
+import logging
+
+from spidermon.exceptions import NotConfigured
+from spidermon.contrib.actions.templates import ActionWithTemplates
+
+logger = logging.getLogger(__name__)
+
+
+class SendSNSNotification(ActionWithTemplates):
+    aws_access_key = None
+    aws_secret_key = None
+    aws_region_name = "us-east-1"
+    topic_arn = None
+
+    def __init__(
+        self,
+        topic_arn=None,
+        aws_access_key=None,
+        aws_secret_key=None,
+        aws_region_name=None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+
+        self.topic_arn = topic_arn or self.topic_arn
+        self.aws_access_key = aws_access_key or self.aws_access_key
+        self.aws_secret_key = aws_secret_key or self.aws_secret_key
+        self.aws_region_name = aws_region_name or self.aws_region_name
+
+        if not self.topic_arn:
+            raise NotConfigured(
+                "You must provide a value for SPIDERMON_SNS_TOPIC_ARN setting."
+            )
+        if not self.aws_access_key:
+            raise NotConfigured(
+                "You must provide a value for SPIDERMON_AWS_ACCESS_KEY_ID setting."
+            )
+        if not self.aws_secret_key:
+            raise NotConfigured(
+                "You must provide a value for SPIDERMON_AWS_SECRET_ACCESS_KEY setting."
+            )
+
+    def run_action(self):
+        self.send_message()
+
+    def send_message(self, subject, attributes):
+        client = boto3.client(
+            service_name="sns",
+            region_name=self.aws_region_name,
+            aws_access_key_id=self.aws_access_key,
+            aws_secret_access_key=self.aws_secret_key,
+        )
+        logger.info(
+            f"Sending SNS message with subject: {subject} and attributes: {attributes}"
+        )
+        try:
+            client.publish(
+                TopicArn=self.topic_arn, Message=subject, MessageAttributes=attributes
+            )
+        except Exception as e:
+            logger.error(f"Failed to send SNS message: {e}")
+            raise
+        logger.info(f"SNS message sent successfully!")
+
+    @classmethod
+    def from_crawler_kwargs(cls, crawler):
+        return {
+            "topic_arn": crawler.settings.get("SPIDERMON_SNS_TOPIC_ARN"),
+            "aws_access_key": crawler.settings.get("SPIDERMON_AWS_ACCESS_KEY_ID"),
+            "aws_secret_key": crawler.settings.get("SPIDERMON_AWS_SECRET_ACCESS_KEY"),
+        }

spidermon/contrib/actions/sns/notifiers.py

@@ -0,0 +1,33 @@
+from . import SendSNSNotification
+
+
+class SendSNSNotificationSpiderStarted(SendSNSNotification):
+    def run_action(self):
+        subject = "Spider Started"
+        attributes = {
+            "EventType": {"DataType": "String", "StringValue": "SpiderStarted"},
+            "SpiderName": {"DataType": "String", "StringValue": self.data.spider.name},
+            "StartTime": {
+                "DataType": "String",
+                "StringValue": str(self.data.stats.start_time),
+            },
+        }
+        self.send_message(subject, attributes)
+
+
+class SendSNSNotificationSpiderFinished(SendSNSNotification):
+    def run_action(self):
+        subject = "Spider Finished"
+        attributes = {
+            "EventType": {"DataType": "String", "StringValue": "SpiderFinished"},
+            "SpiderName": {"DataType": "String", "StringValue": self.data.spider.name},
+            "ItemsScraped": {
+                "DataType": "Number",
+                "StringValue": str(self.data.stats.item_scraped_count),
+            },
+            "FinishTime": {
+                "DataType": "String",
+                "StringValue": str(self.data.stats.finish_time),
+            },
+        }
+        self.send_message(subject, attributes)

tests/contrib/actions/sns/test_sns_action.py

@@ -0,0 +1,120 @@
+import pytest
+
+from scrapy.utils.test import get_crawler
+from spidermon.exceptions import NotConfigured
+from spidermon.contrib.actions.sns import SendSNSNotification
+from spidermon.contrib.actions.sns.notifiers import (
+    SendSNSNotificationSpiderFinished,
+    SendSNSNotificationSpiderStarted,
+)
+
+
+@pytest.fixture
+def boto3_client(mocker):
+    return mocker.patch("spidermon.contrib.actions.sns.boto3.client")
+
+
+@pytest.fixture
+def logger_info(mocker):
+    return mocker.patch("spidermon.contrib.actions.sns.logger.info")
+
+
+@pytest.fixture
+def logger_error(mocker):
+    return mocker.patch("spidermon.contrib.actions.sns.logger.error")
+
+
+@pytest.fixture
+def mock_notifier_data(mocker):
+    data = mocker.MagicMock()
+    data.spider.name = "TestSpider"
+    data.stats.start_time = "2023-08-25 10:00:00"
+    data.stats.finish_time = "2023-08-25 11:00:00"
+    data.stats.item_scraped_count = 100
+    return data
+
+
+def test_fail_if_no_topic_arn():
+    with pytest.raises(NotConfigured):
+        SendSNSNotification(topic_arn=None)
+
+
+def test_fail_if_no_aws_access_key():
+    with pytest.raises(NotConfigured):
+        SendSNSNotification(topic_arn="arn:aws:sns:us-east-1:123456789012:MyTopic")
+
+
+def test_fail_if_no_aws_secret_key():
+    with pytest.raises(NotConfigured):
+        SendSNSNotification(
+            topic_arn="arn:aws:sns:us-east-1:123456789012:MyTopic",
+            aws_access_key="ACCESS_KEY",
+        )
+
+
+def test_send_message(boto3_client, logger_info):
+    notifier = SendSNSNotification(
+        topic_arn="arn:aws:sns:us-east-1:123456789012:MyTopic",
+        aws_access_key="ACCESS_KEY",
+        aws_secret_key="SECRET_KEY",
+    )
+    subject = "Test Notification"
+    attributes = {"EventType": {"DataType": "String", "StringValue": "TestEvent"}}
+    notifier.send_message(subject, attributes)
+    assert boto3_client.call_count == 1
+    assert logger_info.call_count == 2
+
+
+def test_log_error_when_sns_returns_error(boto3_client, logger_error):
+    boto3_client.return_value.publish.side_effect = Exception("SNS Error")
+    notifier = SendSNSNotification(
+        topic_arn="arn:aws:sns:us-east-1:123456789012:MyTopic",
+        aws_access_key="ACCESS_KEY",
+        aws_secret_key="SECRET_KEY",
+    )
+    subject = "Test Notification"
+    attributes = {"EventType": {"DataType": "String", "StringValue": "TestEvent"}}
+    with pytest.raises(Exception, match="SNS Error"):
+        notifier.send_message(subject, attributes)
+    assert logger_error.call_count == 1
+
+
+def test_send_sns_notification_spider_started(mocker, mock_notifier_data):
+    notifier = SendSNSNotificationSpiderStarted(
+        topic_arn="arn:aws:sns:us-east-1:123456789012:MyTopic",
+        aws_access_key="ACCESS_KEY",
+        aws_secret_key="SECRET_KEY",
+    )
+    notifier.data = mock_notifier_data
+
+    expected_subject = "Spider Started"
+    expected_attributes = {
+        "EventType": {"DataType": "String", "StringValue": "SpiderStarted"},
+        "SpiderName": {"DataType": "String", "StringValue": "TestSpider"},
+        "StartTime": {"DataType": "String", "StringValue": "2023-08-25 10:00:00"},
+    }
+
+    mock_send_message = mocker.patch.object(notifier, "send_message")
+    notifier.run_action()
+    mock_send_message.assert_called_once_with(expected_subject, expected_attributes)
+
+
+def test_send_sns_notification_spider_finished(mocker, mock_notifier_data):
+    notifier = SendSNSNotificationSpiderFinished(
+        topic_arn="arn:aws:sns:us-east-1:123456789012:MyTopic",
+        aws_access_key="ACCESS_KEY",
+        aws_secret_key="SECRET_KEY",
+    )
+    notifier.data = mock_notifier_data
+
+    expected_subject = "Spider Finished"
+    expected_attributes = {
+        "EventType": {"DataType": "String", "StringValue": "SpiderFinished"},
+        "SpiderName": {"DataType": "String", "StringValue": "TestSpider"},
+        "ItemsScraped": {"DataType": "Number", "StringValue": "100"},
+        "FinishTime": {"DataType": "String", "StringValue": "2023-08-25 11:00:00"},
+    }
+
+    mock_send_message = mocker.patch.object(notifier, "send_message")
+    notifier.run_action()
+    mock_send_message.assert_called_once_with(expected_subject, expected_attributes)