content dvm optimizations

nostr_dvm/dvm.py

@@ -5,7 +5,7 @@
 from sys import platform
 
 from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \
-    init_logger, LogLevel, Options, nip04_encrypt, NostrSigner, Kind
+    init_logger, LogLevel, Options, nip04_encrypt, NostrSigner, Kind, RelayLimits
 
 import time
 
@@ -39,8 +39,9 @@ def __init__(self, dvm_config, admin_config=None):
         self.keys = Keys.parse(dvm_config.PRIVATE_KEY)
         wait_for_send = False
         skip_disconnected_relays = True
-        opts = (Options().wait_for_send(wait_for_send).send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT))
-                .skip_disconnected_relays(skip_disconnected_relays))
+        relaylimits = RelayLimits.disable()
+        opts = (Options().wait_for_send(wait_for_send). send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT))
+                .skip_disconnected_relays(skip_disconnected_relays).relay_limits(relaylimits))
 
         signer = NostrSigner.keys(self.keys)
         self.client = Client.with_opts(signer, opts)
@@ -690,5 +691,22 @@ def do_work(job_event, amount):
 
                 if Timestamp.now().as_secs() > job.timestamp + 60 * 20:  # remove jobs to look for after 20 minutes..
                     self.jobs_on_hold_list.remove(job)
+            advanced_log = False
+            if advanced_log:
+                for url, relay in self.client.relays().items():
+                    stats = relay.stats()
+                    print(f"Relay: {url}")
+                    print(f"Connected: {relay.is_connected()}")
+                    print(f"Status: {relay.status()}")
+                    print("Stats:")
+                    print(f"    Attempts: {stats.attempts()}")
+                    print(f"    Success: {stats.success()}")
+                    print(f"    Bytes sent: {stats.bytes_sent()}")
+                    print(f"    Bytes received: {stats.bytes_received()}")
+                    print(f"    Connected at: {stats.connected_at().to_human_datetime()}")
+                    if stats.latency() is not None:
+                        print(f"    Latency: {stats.latency().total_seconds() * 1000} ms")
+
+                    print("###########################################")
 
             time.sleep(1.0)

nostr_dvm/tasks/content_discovery_currently_popular.py

@@ -2,7 +2,8 @@
 import os
 from datetime import timedelta
 from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey, NostrSigner, NostrDatabase, \
-    ClientBuilder, Filter, NegentropyOptions, NegentropyDirection, init_logger, LogLevel, Event, EventId, Kind
+    ClientBuilder, Filter, NegentropyOptions, NegentropyDirection, init_logger, LogLevel, Event, EventId, Kind, \
+    RelayOptions
 
 from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface, process_venv
 from nostr_dvm.utils import definitions
@@ -58,20 +59,22 @@ def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, nip88c
             if self.logger:
                 init_logger(LogLevel.DEBUG)
 
-        opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT)))
+        opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT)))
         sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY)
         keys = Keys.parse(sk.to_hex())
         signer = NostrSigner.keys(keys)
         database = NostrDatabase.sqlite(self.db_name)
         self.client = ClientBuilder().signer(signer).database(database).opts(opts).build()
 
-        self.client.add_relay("wss://relay.damus.io")
-        self.client.add_relay("wss://nostr.oxtr.dev")
-        self.client.add_relay("wss://nostr21.com")
+        ropts = RelayOptions().ping(False)
+        self.client.add_relay_with_opts("wss://relay.damus.io", ropts)
+        self.client.add_relay_with_opts("wss://nostr.oxtr.dev", ropts)
+        self.client.add_relay_with_opts("wss://nostr21.com", ropts)
 
         self.client.connect()
 
 
+
         if self.dvm_config.UPDATE_DATABASE:
             self.sync_db()
 
@@ -142,6 +145,9 @@ def calculate_result(self, request_form):
                 if len(reactions) >= self.min_reactions:
                     ns.finallist[event.id().to_hex()] = len(reactions)
 
+        if len(ns.finallist) == 0:
+            return self.result
+
         result_list = []
         finallist_sorted = sorted(ns.finallist.items(), key=lambda x: x[1], reverse=True)[:int(options["max_results"])]
         for entry in finallist_sorted:

nostr_dvm/tasks/content_discovery_currently_popular_by_top_zaps.py

@@ -2,7 +2,8 @@
 import os
 from datetime import timedelta
 from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey, NostrSigner, NostrDatabase, \
-    ClientBuilder, Filter, NegentropyOptions, NegentropyDirection, init_logger, LogLevel, Event, EventId, Kind
+    ClientBuilder, Filter, NegentropyOptions, NegentropyDirection, init_logger, LogLevel, Event, EventId, Kind, \
+    RelayOptions
 
 from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface, process_venv
 from nostr_dvm.utils import definitions
@@ -59,16 +60,17 @@ def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, nip88c
             if self.logger:
                 init_logger(LogLevel.DEBUG)
 
-        opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT)))
+        opts = (Options().wait_for_send(False).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT)))
         sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY)
         keys = Keys.parse(sk.to_hex())
         signer = NostrSigner.keys(keys)
         database = NostrDatabase.sqlite(self.db_name)
         self.client = ClientBuilder().signer(signer).database(database).opts(opts).build()
 
-        self.client.add_relay("wss://relay.damus.io")
-        self.client.add_relay("wss://nostr.oxtr.dev")
-        self.client.add_relay("wss://nostr21.com")
+        ropts = RelayOptions().ping(False)
+        self.client.add_relay_with_opts("wss://relay.damus.io", ropts)
+        self.client.add_relay_with_opts("wss://nostr.oxtr.dev", ropts)
+        self.client.add_relay_with_opts("wss://nostr21.com", ropts)
 
         self.client.connect()
 

nostr_dvm/tasks/content_discovery_currently_popular_followers.py

@@ -58,12 +58,13 @@ def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, nip88c
         database = NostrDatabase.sqlite(self.db_name)
         self.client = ClientBuilder().signer(signer).database(database).opts(opts).build()
 
-        self.client.add_relay("wss://relay.damus.io")
-        self.client.add_relay("wss://nostr.oxtr.dev")
-        self.client.add_relay("wss://nostr21.com")
-
         ropts = RelayOptions().ping(False)
-        self.client.add_relay_with_opts("wss://nostr.band", ropts)
+        self.client.add_relay_with_opts("wss://relay.damus.io", ropts)
+        self.client.add_relay_with_opts("wss://nostr.oxtr.dev", ropts)
+        self.client.add_relay_with_opts("wss://nostr21.com", ropts)
+
+        #ropts = RelayOptions().ping(False)
+        #self.client.add_relay_with_opts("wss://nostr.band", ropts)
 
         self.client.connect()
 
@@ -114,7 +115,7 @@ def process(self, request_form):
 
         relaylimits = RelayLimits.disable()
         opts = (
-            Options().wait_for_send(True).send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT)).relay_limits(
+            Options().wait_for_send(True).send_timeout(timedelta(seconds=self.dvm_config.RELAY_LONG_TIMEOUT)).relay_limits(
                 relaylimits))
 
         user = PublicKey.parse(options["user"])

nostr_dvm/tasks/content_discovery_currently_popular_topic.py

@@ -2,7 +2,8 @@
 import os
 from datetime import timedelta
 from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey, NostrSigner, NostrDatabase, \
-    ClientBuilder, Filter, NegentropyOptions, NegentropyDirection, init_logger, LogLevel, Event, EventId, Kind
+    ClientBuilder, Filter, NegentropyOptions, NegentropyDirection, init_logger, LogLevel, Event, EventId, Kind, \
+    RelayLimits, RelayOptions
 
 from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface, process_venv
 from nostr_dvm.utils import definitions
@@ -73,16 +74,23 @@ def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, nip88c
             if self.logger:
                 init_logger(LogLevel.DEBUG)
 
-        opts = (Options().wait_for_send(True).send_timeout(timedelta(seconds=self.dvm_config.RELAY_TIMEOUT)))
+        relaylimits = RelayLimits.disable()
+        opts = (Options().wait_for_send(True).send_timeout(timedelta(seconds=dvm_config.RELAY_LONG_TIMEOUT))
+                .relay_limits(relaylimits))
         sk = SecretKey.from_hex(self.dvm_config.PRIVATE_KEY)
         keys = Keys.parse(sk.to_hex())
         signer = NostrSigner.keys(keys)
         database = NostrDatabase.sqlite(self.db_name)
         self.client = ClientBuilder().signer(signer).database(database).opts(opts).build()
 
-        self.client.add_relay("wss://relay.damus.io")
-        self.client.add_relay("wss://nostr.oxtr.dev")
-        self.client.add_relay("wss://nostr21.com")
+        #self.client.add_relay("wss://relay.damus.io")
+        #self.client.add_relay("wss://nostr.oxtr.dev")
+
+        ropts = RelayOptions().ping(False)
+        self.client.add_relay_with_opts("wss://relay.damus.io", ropts)
+        self.client.add_relay_with_opts("wss://nostr.oxtr.dev", ropts)
+        self.client.add_relay_with_opts("wss://nostr21.com", ropts)
+
         self.client.connect()
 
         if self.dvm_config.UPDATE_DATABASE:

nostr_dvm/utils/dvmconfig.py

@@ -16,12 +16,15 @@ class DVMConfig:
     FIX_COST: float = None
     PER_UNIT_COST: float = None
 
-    RELAY_LIST = ["wss://dvms.f7z.io", "wss://relay.damus.io", "wss://nostr.wine",
+    RELAY_LIST = ["wss://dvms.f7z.io",
                   "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
                   "wss://relay.nostr.net"
                   ]
 
+    #"wss://relay.damus.io"
+
     RELAY_TIMEOUT = 5
+    RELAY_LONG_TIMEOUT = 30
     EXTERNAL_POST_PROCESS_TYPE = PostProcessFunctionType.NONE  # Leave this on None, except the DVM is external
     LNBITS_INVOICE_KEY = ''  # Will all automatically generated by default, or read from .env
     LNBITS_ADMIN_KEY = ''  # In order to pay invoices, e.g. from the bot to DVMs, or reimburse users.

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '0.5.3'
+VERSION = '0.5.4'
 DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines'
 LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. See the github repository for more information')
 

tests/discovery.py

@@ -19,18 +19,22 @@
 from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
 
 rebbroadcast_NIP89 = False   # Announce NIP89 on startup
-global_update_rate = 180     # set this high on first sync so db can fully sync before another process trys to.
-use_logger = False
+global_update_rate = 300     # set this high on first sync so db can fully sync before another process trys to.
+use_logger = True
 
 #git_hash = NostrLibrary().git_hash_version()
 #print("GitHash " + git_hash)
 
 if use_logger:
-    init_logger(LogLevel.DEBUG)
+    init_logger(LogLevel.INFO)
 def build_example_nostrband(name, identifier, admin_config, image, about, custom_processing_msg):
     dvm_config: DVMConfig = build_default_config(identifier)
     dvm_config.USE_OWN_VENV = False
     dvm_config.CUSTOM_PROCESSING_MESSAGE = custom_processing_msg
+    dvm_config.RELAY_LIST = ["wss://dvms.f7z.io",
+                  "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
+                  "wss://relay.nostr.net"
+                  ]
     admin_config.LUD16 = dvm_config.LN_ADDRESS
     # Add NIP89
 
@@ -61,6 +65,10 @@ def build_example_topic(name, identifier, admin_config, options, image, descript
     dvm_config.UPDATE_DATABASE = update_db
     dvm_config.FIX_COST = cost
     dvm_config.CUSTOM_PROCESSING_MESSAGE = processing_msg
+    dvm_config.RELAY_LIST = ["wss://dvms.f7z.io",
+                             "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
+                             "wss://relay.nostr.net"
+                             ]
     admin_config.LUD16 = dvm_config.LN_ADDRESS
 
     # Add NIP89
@@ -99,7 +107,13 @@ def build_example_popular(name, identifier, admin_config, options, image, cost=0
     dvm_config.SCHEDULE_UPDATES_SECONDS = update_rate  # Every 10 minutes
     dvm_config.UPDATE_DATABASE = update_db
     dvm_config.FIX_COST = cost
+    #dvm_config.RELAY_LIST = ["wss://dvms.f7z.io", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
+    #"wss://relay.nostr.net"]
     dvm_config.CUSTOM_PROCESSING_MESSAGE = processing_msg
+    dvm_config.RELAY_LIST = ["wss://dvms.f7z.io",
+                             "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
+                             "wss://relay.nostr.net"
+                             ]
     admin_config.LUD16 = dvm_config.LN_ADDRESS
 
     # Add NIP89
@@ -138,6 +152,10 @@ def build_example_popular_followers(name, identifier, admin_config, options, ima
     dvm_config.UPDATE_DATABASE = update_db
     dvm_config.FIX_COST = cost
     dvm_config.CUSTOM_PROCESSING_MESSAGE = processing_msg
+    dvm_config.RELAY_LIST = ["wss://dvms.f7z.io",
+                             "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
+                             "wss://relay.nostr.net"
+                             ]
     admin_config.LUD16 = dvm_config.LN_ADDRESS
 
     # Add NIP89
@@ -179,6 +197,10 @@ def build_example_top_zapped(name, identifier, admin_config, options, image, cos
     dvm_config.UPDATE_DATABASE = update_db
     dvm_config.FIX_COST = cost
     dvm_config.CUSTOM_PROCESSING_MESSAGE = processing_msg
+    dvm_config.RELAY_LIST = ["wss://dvms.f7z.io",
+                             "wss://nostr.mom", "wss://nostr.oxtr.dev", "wss://relay.nostr.bg",
+                             "wss://relay.nostr.net"
+                             ]
     admin_config.LUD16 = dvm_config.LN_ADDRESS
 
     # Add NIP89
@@ -340,8 +362,8 @@ def playground():
     # Popular top zapped
     admin_config_top_zaps = AdminConfig()
     admin_config_top_zaps.REBROADCAST_NIP89 = rebbroadcast_NIP89
-    admin_config_top_zaps.UPDATE_PROFILE = False
-    #admin_config_top_zaps.DELETE_NIP89 = True
+    #admin_config_top_zaps.UPDATE_PROFILE = False
+    #admin_config_top_zaps.DELETE_NIP89 = False
     #admin_config_top_zaps.PRIVKEY = ""
     #admin_config_top_zaps.EVENTID = ""
     #admin_config_top_zaps.POW = True
@@ -371,10 +393,10 @@ def playground():
     admin_config_followers = AdminConfig()
     admin_config_followers.REBROADCAST_NIP89 = rebbroadcast_NIP89
     admin_config_followers.UPDATE_PROFILE = False
-    #admin_config_followers.DELETE_NIP89 = True
-    #admin_config_followers.PRIVKEY = ""
-    #admin_config_followers.EVENTID = ""
-    #admin_config_followers.POW = True
+    admin_config_followers.DELETE_NIP89 = False
+    admin_config_followers.PRIVKEY = ""
+    admin_config_followers.EVENTID = ""
+    admin_config_followers.POW = True
     custom_processing_msg = ["Processing popular notes from npubs you follow..",
                              "Let's see what npubs you follow have been up to..",
                              "Processing a personalized feed, just for you.."]