From 6792fbc7dd0b0cb68a734c99fa8b44c1746e8684 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sat, 16 Dec 2023 00:42:06 +0100 Subject: [PATCH 01/16] update --- .gitignore | 1 + nostr_dvm/dvm.py | 29 ++++++-- nostr_dvm/interfaces/dvmtaskinterface.py | 52 +++++++++----- nostr_dvm/tasks/advanced_search.py | 37 ++++++---- nostr_dvm/tasks/convert_media.py | 36 ++++++---- nostr_dvm/tasks/discovery_inactive_follows.py | 34 ++++++---- .../tasks/imagegeneration_openai_dalle.py | 45 +++++++++---- .../tasks/imagegeneration_replicate_sdxl.py | 42 +++++++----- nostr_dvm/tasks/textextraction_google.py | 39 ++++++----- nostr_dvm/tasks/textextraction_pdf.py | 37 ++++++---- nostr_dvm/tasks/textgeneration_llmlite.py | 38 +++++++---- nostr_dvm/tasks/translation_google.py | 44 ++++++++---- nostr_dvm/tasks/translation_libretranslate.py | 67 +++++++++++-------- nostr_dvm/tasks/trending_notes_nostrband.py | 39 ++++++----- .../tasks/videogeneration_replicate_svd.py | 46 ++++++++----- nostr_dvm/utils/dvmconfig.py | 3 + tests/test_dvm_client.py | 8 +-- 17 files changed, 385 insertions(+), 212 deletions(-) diff --git a/.gitignore b/.gitignore index 54c9476..5bb30f3 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,4 @@ db/* backends/nserver/venv backends/nserver/cache backends/nserver/modules/image_upscale/weights +cache/venvs/ diff --git a/nostr_dvm/dvm.py b/nostr_dvm/dvm.py index 91ede4e..27c09ca 100644 --- a/nostr_dvm/dvm.py +++ b/nostr_dvm/dvm.py @@ -1,5 +1,9 @@ +import importlib import json +import os +import subprocess from datetime import timedelta +from pathlib import Path from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \ init_logger, LogLevel, Options, nip04_encrypt @@ -323,7 +327,8 @@ class DVM: except Exception as e: # Zapping back by error in post-processing is a risk for the DVM because work has been done, # but maybe something with parsing/uploading failed. Try to avoid errors here as good as possible - send_job_status_reaction(original_event, "error", content="Error in Post-processing: " + str(e), + send_job_status_reaction(original_event, "error", + content="Error in Post-processing: " + str(e), dvm_config=self.dvm_config, ) if amount > 0 and self.dvm_config.LNBITS_ADMIN_KEY != "": @@ -466,11 +471,24 @@ class DVM: for dvm in self.dvm_config.SUPPORTED_DVMS: try: if task == dvm.TASK: - request_form = dvm.create_request_from_nostr_event(job_event, self.client, - self.dvm_config) - result = dvm.process(request_form) + + request_form = dvm.create_request_from_nostr_event(job_event, self.client, self.dvm_config) + python_bin = (r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] + + "/bin/python") + subprocess.run([python_bin, dvm_config.SCRIPT, + '--request', json.dumps(request_form), + '--identifier', dvm_config.IDENTIFIER, + '--output', 'output.txt']) + + print("Finished processing, loading data..") + + with open(os.path.abspath('output.txt')) as f: + result = f.readlines()[0] + print(result) + #f.close() + os.remove(os.path.abspath('output.txt')) try: - post_processed = dvm.post_process(result, job_event) + post_processed = dvm.post_process(str(result), job_event) send_nostr_reply_event(post_processed, job_event.as_json()) except Exception as e: send_job_status_reaction(job_event, "error", content=str(e), @@ -495,7 +513,6 @@ class DVM: except Exception as e: print(e) - return self.client.handle_notifications(NotificationHandler()) diff --git a/nostr_dvm/interfaces/dvmtaskinterface.py b/nostr_dvm/interfaces/dvmtaskinterface.py index 8525f6f..83125d7 100644 --- a/nostr_dvm/interfaces/dvmtaskinterface.py +++ b/nostr_dvm/interfaces/dvmtaskinterface.py @@ -1,18 +1,16 @@ import json import os import subprocess +from subprocess import run import sys from threading import Thread - +from venv import create from nostr_sdk import Keys - from nostr_dvm.dvm import DVM from nostr_dvm.utils.admin_utils import AdminConfig from nostr_dvm.utils.dvmconfig import DVMConfig from nostr_dvm.utils.nip89_utils import NIP89Config -from nostr_dvm.utils.nostr_utils import check_and_set_private_key from nostr_dvm.utils.output_utils import post_process_result -from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys class DVMTaskInterface: @@ -30,11 +28,13 @@ class DVMTaskInterface: admin_config: AdminConfig dependencies = [] + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None, task=None): self.init(name, dvm_config, admin_config, nip89config, task) self.options = options - self.install_dependencies(self.dependencies) + self.make_venv(dvm_config) + def init(self, name, dvm_config, admin_config=None, nip89config=None, task=None): self.NAME = name @@ -58,6 +58,16 @@ class DVMTaskInterface: self.dvm_config = dvm_config self.admin_config = admin_config + def make_venv(self, dvm_config): + if dvm_config.SCRIPT != "": + dir = r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] + if not os.path.isdir(dir): + print(dir) + create(dir, with_pip=True, upgrade_deps=True) + for (module, package) in self.dependencies: + print("Installing Module: " + module) + run(["bin/pip", "install", package], cwd=dir) + def run(self): nostr_dvm_thread = Thread(target=self.DVM, args=[self.dvm_config, self.admin_config]) nostr_dvm_thread.start() @@ -87,16 +97,6 @@ class DVMTaskInterface: """Post-process the data and return the result Use default function, if not overwritten""" return post_process_result(result, event) - def install_dependencies(self, packages): - import pip - for module, package in packages: - try: - __import__(module) - except ImportError: - subprocess.check_call([sys.executable, "-m", "pip", "install", package]) - - - @staticmethod def set_options(request_form): print("Setting options...") @@ -105,3 +105,25 @@ class DVMTaskInterface: opts = json.loads(request_form["options"]) print(opts) return dict(opts) + + @staticmethod + def process_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--request', dest='request') + parser.add_argument('--identifier', dest='identifier') + parser.add_argument('--output', dest='output') + args = parser.parse_args() + return args + + @staticmethod + def write_output(result, output): + with open(os.path.abspath(output), 'w') as f: + f.write(result) + #f.close() + + def process_venv(self): + pass + + if __name__ == '__main__': + process_venv() diff --git a/nostr_dvm/tasks/advanced_search.py b/nostr_dvm/tasks/advanced_search.py index 793d304..7bd2816 100644 --- a/nostr_dvm/tasks/advanced_search.py +++ b/nostr_dvm/tasks/advanced_search.py @@ -27,9 +27,11 @@ class AdvancedSearch(DVMTaskInterface): TASK: str = "search-content" FIX_COST: float = 0 dvm_config: DVMConfig + dependencies = [("nostr-dvm", "nostr-dvm")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -177,19 +179,28 @@ def build_example(name, identifier, admin_config): admin_config=admin_config) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = AdvancedSearch(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False +#if __name__ == '__main__': +# process_venv() - dvm = build_example("Advanced Nostr Search", "discovery_content_search", admin_config) - dvm.run() - keep_alive() + #if env_path.is_file(): + # print(f'loading environment from {env_path.resolve()}') + # dotenv.load_dotenv(env_path, verbose=True, override=True) + #else: + # raise FileNotFoundError(f'.env file not found at {env_path} ') + # + #admin_config = AdminConfig() + #admin_config.REBROADCAST_NIP89 = False + #admin_config.UPDATE_PROFILE = False + + #dvm = build_example("Advanced Nostr Search", "discovery_content_search", admin_config) + #dvm.run() + + #keep_alive() diff --git a/nostr_dvm/tasks/convert_media.py b/nostr_dvm/tasks/convert_media.py index ef1666c..20ff106 100644 --- a/nostr_dvm/tasks/convert_media.py +++ b/nostr_dvm/tasks/convert_media.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path import dotenv @@ -30,6 +31,7 @@ class MediaConverter(DVMTaskInterface): def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -106,20 +108,28 @@ def build_example(name, identifier, admin_config): return MediaConverter(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = MediaConverter(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +#if __name__ == '__main__': +# process_venv() - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False + #env_path = Path('.env') + #if env_path.is_file(): + # print(f'loading environment from {env_path.resolve()}') + # dotenv.load_dotenv(env_path, verbose=True, override=True) + #else: + # raise FileNotFoundError(f'.env file not found at {env_path} ') - dvm = build_example("Media Bringer", "media_converter", admin_config) - dvm.run() + #admin_config = AdminConfig() + #admin_config.REBROADCAST_NIP89 = False + #admin_config.UPDATE_PROFILE = False - keep_alive() + #dvm = build_example("Media Bringer", "media_converter", admin_config) + #dvm.run() + + #keep_alive() diff --git a/nostr_dvm/tasks/discovery_inactive_follows.py b/nostr_dvm/tasks/discovery_inactive_follows.py index 08a0aeb..6a52df5 100644 --- a/nostr_dvm/tasks/discovery_inactive_follows.py +++ b/nostr_dvm/tasks/discovery_inactive_follows.py @@ -33,6 +33,7 @@ class DiscoverInactiveFollows(DVMTaskInterface): def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -198,19 +199,26 @@ def build_example(name, identifier, admin_config): return DiscoverInactiveFollows(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = DiscoverInactiveFollows(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +#if __name__ == '__main__': +# process_venv() + #env_path = Path('.env') + #if env_path.is_file(): + # print(f'loading environment from {env_path.resolve()}') + # dotenv.load_dotenv(env_path, verbose=True, override=True) + #else: + # raise FileNotFoundError(f'.env file not found at {env_path} ') - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("Bygones", "discovery_inactive_follows", admin_config) - dvm.run() + #admin_config = AdminConfig() + #admin_config.REBROADCAST_NIP89 = False + #admin_config.UPDATE_PROFILE = False + #dvm = build_example("Bygones", "discovery_inactive_follows", admin_config) + #dvm.run() - keep_alive() + #keep_alive() diff --git a/nostr_dvm/tasks/imagegeneration_openai_dalle.py b/nostr_dvm/tasks/imagegeneration_openai_dalle.py index 920bfa0..3412784 100644 --- a/nostr_dvm/tasks/imagegeneration_openai_dalle.py +++ b/nostr_dvm/tasks/imagegeneration_openai_dalle.py @@ -1,5 +1,6 @@ import json import os +import time from io import BytesIO from pathlib import Path @@ -28,10 +29,12 @@ class ImageGenerationDALLE(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE TASK: str = "text-to-image" FIX_COST: float = 120 - dependencies = [("openai", "openai==1.3.5")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("openai", "openai==1.3.5")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -107,6 +110,7 @@ class ImageGenerationDALLE(DVMTaskInterface): n=int(options['number']), ) + image_url = response.data[0].url # rehost the result instead of relying on the openai link response = requests.get(image_url) @@ -151,18 +155,31 @@ def build_example(name, identifier, admin_config): return ImageGenerationDALLE(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = ImageGenerationDALLE(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = "" + while result == "": + result = dvm.process(json.loads(args.request)) + time.sleep(10) - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("Dall-E 3", "dalle3", admin_config) - dvm.run() - keep_alive() + DVMTaskInterface.write_output(result, args.output) + +#if __name__ == '__main__': +# process_venv() + #env_path = Path('.env') + #if env_path.is_file(): + # print(f'loading environment from {env_path.resolve()}') + # dotenv.load_dotenv(env_path, verbose=True, override=True) + #else: + # raise FileNotFoundError(f'.env file not found at {env_path} ') + + #admin_config = AdminConfig() + #admin_config.REBROADCAST_NIP89 = False + #admin_config.UPDATE_PROFILE = False + #dvm = build_example("Dall-E 3", "dalle3", admin_config) + #dvm.run() + + #keep_alive() diff --git a/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py b/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py index 480d0ba..0aa7ab0 100644 --- a/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py +++ b/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py @@ -1,15 +1,11 @@ import json import os from io import BytesIO -from pathlib import Path - -import dotenv import requests from PIL import Image from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig -from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag @@ -29,10 +25,12 @@ class ImageGenerationReplicateSDXL(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE TASK: str = "text-to-image" FIX_COST: float = 120 - dependencies = [("replicate", "replicate==0.21.1")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("replicate", "replicate==0.21.1")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -147,18 +145,26 @@ def build_example(name, identifier, admin_config): admin_config=admin_config) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = ImageGenerationReplicateSDXL(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("Stable Diffusion XL", "replicate_sdxl", admin_config) - dvm.run() +#if __name__ == '__main__': +# process_venv() + #env_path = Path('.env') + #if env_path.is_file(): + # print(f'loading environment from {env_path.resolve()}') + # dotenv.load_dotenv(env_path, verbose=True, override=True) + #else: + # raise FileNotFoundError(f'.env file not found at {env_path} ') - keep_alive() + #admin_config = AdminConfig() + #admin_config.REBROADCAST_NIP89 = False + #admin_config.UPDATE_PROFILE = False + #dvm = build_example("Stable Diffusion XL", "replicate_sdxl", admin_config) + #dvm.run() + + #keep_alive() diff --git a/nostr_dvm/tasks/textextraction_google.py b/nostr_dvm/tasks/textextraction_google.py index cbae2c3..2c24ef7 100644 --- a/nostr_dvm/tasks/textextraction_google.py +++ b/nostr_dvm/tasks/textextraction_google.py @@ -27,10 +27,13 @@ class SpeechToTextGoogle(DVMTaskInterface): TASK: str = "speech-to-text" FIX_COST: float = 10 PER_UNIT_COST: float = 0.1 - dependencies = [("speech_recognition", "SpeechRecognition==3.10.0")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("speech_recognition", "SpeechRecognition==3.10.0")] + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) if options is None: options = {} @@ -155,20 +158,26 @@ def build_example(name, identifier, admin_config): return SpeechToTextGoogle(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, options=options) +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = SpeechToTextGoogle(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +#if __name__ == '__main__': +# env_path = Path('.env') +# if env_path.is_file(): +# print(f'loading environment from {env_path.resolve()}') +# dotenv.load_dotenv(env_path, verbose=True, override=True) +# else: +# raise FileNotFoundError(f'.env file not found at {env_path} ') +# +# admin_config = AdminConfig() +# admin_config.REBROADCAST_NIP89 = False +# admin_config.UPDATE_PROFILE = False +# dvm = build_example("Transcriptor", "speech_recognition", admin_config) +# dvm.run() - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("Transcriptor", "speech_recognition", admin_config) - dvm.run() - - keep_alive() + # keep_alive() diff --git a/nostr_dvm/tasks/textextraction_pdf.py b/nostr_dvm/tasks/textextraction_pdf.py index e9062cf..8101840 100644 --- a/nostr_dvm/tasks/textextraction_pdf.py +++ b/nostr_dvm/tasks/textextraction_pdf.py @@ -26,11 +26,13 @@ class TextExtractionPDF(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT TASK: str = "pdf-to-text" FIX_COST: float = 0 - dependencies = [("pypdf", "pypdf==3.17.1")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("pypdf", "pypdf==3.17.1")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) @@ -116,19 +118,26 @@ def build_example(name, identifier, admin_config): return TextExtractionPDF(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = TextExtractionPDF(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("PDF Extractor", "pdf_extractor", admin_config) - dvm.run() +#if __name__ == '__main__': +# env_path = Path('.env') +# if env_path.is_file(): +# print(f'loading environment from {env_path.resolve()}') +# dotenv.load_dotenv(env_path, verbose=True, override=True) +# else: +# raise FileNotFoundError(f'.env file not found at {env_path} ') - keep_alive() +# admin_config = AdminConfig() +# admin_config.REBROADCAST_NIP89 = False +# admin_config.UPDATE_PROFILE = False +# dvm = build_example("PDF Extractor", "pdf_extractor", admin_config) +# dvm.run() + +# keep_alive() diff --git a/nostr_dvm/tasks/textgeneration_llmlite.py b/nostr_dvm/tasks/textgeneration_llmlite.py index 0a692e7..38090d7 100644 --- a/nostr_dvm/tasks/textgeneration_llmlite.py +++ b/nostr_dvm/tasks/textgeneration_llmlite.py @@ -24,10 +24,12 @@ class TextGenerationOLLAMA(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_TEXT TASK: str = "text-to-text" FIX_COST: float = 0 - dependencies = [("litellm", "litellm==1.12.3")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("litellm", "litellm==1.12.3")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) @@ -125,19 +127,27 @@ def build_example(name, identifier, admin_config): return TextGenerationOLLAMA(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, options=options) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = TextGenerationOLLAMA(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("LLM", "llmlite", admin_config) - dvm.run() +#if __name__ == '__main__': +# env_path = Path('.env') +# if env_path.is_file(): +# print(f'loading environment from {env_path.resolve()}') +# dotenv.load_dotenv(env_path, verbose=True, override=True) +# else: +# raise FileNotFoundError(f'.env file not found at {env_path} ') +# +# admin_config = AdminConfig() +# admin_config.REBROADCAST_NIP89 = False +# admin_config.UPDATE_PROFILE = False +# +# dvm = build_example("LLM", "llmlite", admin_config) +# dvm.run() - keep_alive() +# keep_alive() diff --git a/nostr_dvm/tasks/translation_google.py b/nostr_dvm/tasks/translation_google.py index 642cf49..e9bcee7 100644 --- a/nostr_dvm/tasks/translation_google.py +++ b/nostr_dvm/tasks/translation_google.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path import dotenv @@ -25,10 +26,12 @@ class TranslationGoogle(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT TASK: str = "translation" FIX_COST: float = 0 - dependencies = [("translatepy", "translatepy==2.3")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("translatepy", "translatepy==2.3")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -141,18 +144,31 @@ def build_example(name, identifier, admin_config): return TranslationGoogle(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("Google Translator", "google_translator", admin_config) - dvm.run() - keep_alive() +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = TranslationGoogle(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) + + + +#if __name__ == '__main__': + #process_venv() + + #env_path = Path('.env') + #if env_path.is_file(): + # print(f'loading environment from {env_path.resolve()}') + # dotenv.load_dotenv(env_path, verbose=True, override=True) + #else: + # raise FileNotFoundError(f'.env file not found at {env_path} ') + + #admin_config = AdminConfig() + #admin_config.REBROADCAST_NIP89 = False + #admin_config.UPDATE_PROFILE = False + #dvm = build_example("Google Translator", "google_translator", admin_config) + #dvm.run() + + #keep_alive() diff --git a/nostr_dvm/tasks/translation_libretranslate.py b/nostr_dvm/tasks/translation_libretranslate.py index 55ab201..9a89c3b 100644 --- a/nostr_dvm/tasks/translation_libretranslate.py +++ b/nostr_dvm/tasks/translation_libretranslate.py @@ -13,7 +13,6 @@ from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag from nostr_dvm.utils.nostr_utils import get_referenced_event_by_id, get_event_by_id - """ This File contains a Module to call Libre Translate Services @@ -32,6 +31,7 @@ class TranslationLibre(DVMTaskInterface): def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None, task=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options, task) def is_input_supported(self, tags): @@ -81,9 +81,9 @@ class TranslationLibre(DVMTaskInterface): def process(self, request_form): options = DVMTaskInterface.set_options(request_form) request = { - "q": options["text"], + "q": options["text"], "source": "auto", - "target": options["language"] + "target": options["language"] } if options["libre_api_key"] != "": request["api_key"] = options["libre_api_key"] @@ -95,10 +95,10 @@ class TranslationLibre(DVMTaskInterface): reply = json.loads(response.text) if reply.get("translatedText"): translated_text = reply['translatedText'] - #untested - #confidence = reply["detectedLanguage"]['confidence'] - #language = reply["detectedLanguage"]['language'] - #print(translated_text + "language: " + language + "conf: " + confidence) + # untested + # confidence = reply["detectedLanguage"]['confidence'] + # language = reply["detectedLanguage"]['language'] + # print(translated_text + "language: " + language + "conf: " + confidence) else: return response.text @@ -125,12 +125,18 @@ def build_example(name, identifier, admin_config): "nip90Params": { "language": { "required": False, - "values": ["en", "az", "be", "bg", "bn", "bs", "ca", "ceb", "co", "cs", "cy", "da", "de", "el", "eo", "es", - "et", "eu", "fa", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "haw", "hi", "hmn", "hr", "ht", - "hu", "hy", "id", "ig", "is", "it", "he", "ja", "jv", "ka", "kk", "km", "kn", "ko", "ku", "ky", - "la", "lb", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "ne", "nl", - "no", "ny", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "sm", "sn", "so", - "sq", "sr", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "tl", "tr", "ug", "uk", "ur", "uz", + "values": ["en", "az", "be", "bg", "bn", "bs", "ca", "ceb", "co", "cs", "cy", "da", "de", "el", "eo", + "es", + "et", "eu", "fa", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "haw", "hi", "hmn", "hr", + "ht", + "hu", "hy", "id", "ig", "is", "it", "he", "ja", "jv", "ka", "kk", "km", "kn", "ko", "ku", + "ky", + "la", "lb", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt", "my", "ne", + "nl", + "no", "ny", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "sm", "sn", + "so", + "sq", "sr", "st", "su", "sv", "sw", "ta", "te", "tg", "th", "tl", "tr", "ug", "uk", "ur", + "uz", "vi", "xh", "yi", "yo", "zh", "zu"] } } @@ -143,18 +149,25 @@ def build_example(name, identifier, admin_config): admin_config=admin_config, options=options) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = TranslationLibre(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - dvm = build_example("Libre Translator", "libre_translator", admin_config) - dvm.run() - - keep_alive() +# if __name__ == '__main__': +# env_path = Path('.env') +# if env_path.is_file(): +# print(f'loading environment from {env_path.resolve()}') +# dotenv.load_dotenv(env_path, verbose=True, override=True) +# else: +# raise FileNotFoundError(f'.env file not found at {env_path} ') +# +# admin_config = AdminConfig() +# admin_config.REBROADCAST_NIP89 = False +# admin_config.UPDATE_PROFILE = False +# dvm = build_example("Libre Translator", "libre_translator", admin_config) +# dvm.run() +# +# keep_alive() diff --git a/nostr_dvm/tasks/trending_notes_nostrband.py b/nostr_dvm/tasks/trending_notes_nostrband.py index 480de1f..b915ed7 100644 --- a/nostr_dvm/tasks/trending_notes_nostrband.py +++ b/nostr_dvm/tasks/trending_notes_nostrband.py @@ -29,6 +29,7 @@ class TrendingNotesNostrBand(DVMTaskInterface): def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) super().__init__(name, dvm_config, nip89config, admin_config, options) def is_input_supported(self, tags): @@ -117,20 +118,26 @@ def build_example(name, identifier, admin_config): return TrendingNotesNostrBand(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = TrendingNotesNostrBand(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') - - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - - dvm = build_example("Trending Notes on Nostr.band", "trending_notes_nostrband", admin_config) - dvm.run() - - keep_alive() +#if __name__ == '__main__': +# env_path = Path('.env') +# if env_path.is_file(): +# print(f'loading environment from {env_path.resolve()}') +# dotenv.load_dotenv(env_path, verbose=True, override=True) +# else: +# raise FileNotFoundError(f'.env file not found at {env_path} ') +# +# admin_config = AdminConfig() +# admin_config.REBROADCAST_NIP89 = False +# admin_config.UPDATE_PROFILE = False +# +# dvm = build_example("Trending Notes on Nostr.band", "trending_notes_nostrband", admin_config) +# dvm.run() +# +# keep_alive() diff --git a/nostr_dvm/tasks/videogeneration_replicate_svd.py b/nostr_dvm/tasks/videogeneration_replicate_svd.py index 9114446..7bf2fe8 100644 --- a/nostr_dvm/tasks/videogeneration_replicate_svd.py +++ b/nostr_dvm/tasks/videogeneration_replicate_svd.py @@ -1,5 +1,6 @@ import json import os +import subprocess from io import BytesIO from pathlib import Path @@ -30,12 +31,18 @@ class VideoGenerationReplicateSVD(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_VIDEO TASK: str = "image-to-video" FIX_COST: float = 120 - dependencies = [("replicate", "replicate==0.21.1")] + dependencies = [("nostr-dvm", "nostr-dvm"), + ("replicate", "replicate==0.21.1")] def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) + print(dvm_config.SCRIPT) super().__init__(name, dvm_config, nip89config, admin_config, options) + + + def is_input_supported(self, tags): for tag in tags: if tag.as_vec()[0] == 'i': @@ -134,20 +141,27 @@ def build_example(name, identifier, admin_config): return VideoGenerationReplicateSVD(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = VideoGenerationReplicateSVD(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) -if __name__ == '__main__': - env_path = Path('.env') - if env_path.is_file(): - print(f'loading environment from {env_path.resolve()}') - dotenv.load_dotenv(env_path, verbose=True, override=True) - else: - raise FileNotFoundError(f'.env file not found at {env_path} ') - admin_config = AdminConfig() - admin_config.REBROADCAST_NIP89 = False - admin_config.UPDATE_PROFILE = False - - dvm = build_example("Stable Video Diffusion", "replicate_svd", admin_config) - dvm.run() - - keep_alive() +#if __name__ == '__main__': +# env_path = Path('.env') +# if env_path.is_file(): +# print(f'loading environment from {env_path.resolve()}') +# dotenv.load_dotenv(env_path, verbose=True, override=True) +# else: +# raise FileNotFoundError(f'.env file not found at {env_path} ') +# +# admin_config = AdminConfig() +# admin_config.REBROADCAST_NIP89 = False +# admin_config.UPDATE_PROFILE = False +# +# dvm = build_example("Stable Video Diffusion", "replicate_svd", admin_config) +# dvm.run() +# +# keep_alive() diff --git a/nostr_dvm/utils/dvmconfig.py b/nostr_dvm/utils/dvmconfig.py index f43cde4..976cd74 100644 --- a/nostr_dvm/utils/dvmconfig.py +++ b/nostr_dvm/utils/dvmconfig.py @@ -27,6 +27,8 @@ class DVMConfig: LNBITS_ADMIN_KEY = '' # In order to pay invoices, e.g. from the bot to DVMs, or reimburse users. LNBITS_URL = 'https://lnbits.com' LN_ADDRESS = '' + SCRIPT = '' + IDENTIFIER = '' DB: str NEW_USER_BALANCE: int = 0 # Free credits for new users NIP89: NIP89Config @@ -36,6 +38,7 @@ class DVMConfig: def build_default_config(identifier): dvm_config = DVMConfig() dvm_config.PRIVATE_KEY = check_and_set_private_key(identifier) + dvm_config.IDENTIFIER = identifier npub = Keys.from_sk_str(dvm_config.PRIVATE_KEY).public_key().to_bech32() invoice_key, admin_key, wallet_id, user_id, lnaddress = check_and_set_ln_bits_keys(identifier, npub) dvm_config.LNBITS_INVOICE_KEY = invoice_key diff --git a/tests/test_dvm_client.py b/tests/test_dvm_client.py index 298066e..7a5dd35 100644 --- a/tests/test_dvm_client.py +++ b/tests/test_dvm_client.py @@ -69,7 +69,7 @@ def nostr_client_test_image(prompt): def nostr_client_test_image_private(prompt, cashutoken): keys = Keys.from_sk_str(check_and_set_private_key("test_client")) - receiver_keys = Keys.from_sk_str(check_and_set_private_key("sketcher")) + receiver_keys = Keys.from_sk_str(check_and_set_private_key("replicate_sdxl")) # TODO more advanced logic, more parsing, params etc, just very basic test functions for now @@ -125,13 +125,13 @@ def nostr_client(): client.subscribe([dm_zap_filter, dvm_filter]) #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) - nostr_client_test_translation("note1p8cx2dz5ss5gnk7c59zjydcncx6a754c0hsyakjvnw8xwlm5hymsnc23rs", "event", "es", 20,20) + #nostr_client_test_translation("note1p8cx2dz5ss5gnk7c59zjydcncx6a754c0hsyakjvnw8xwlm5hymsnc23rs", "event", "es", 20,20) #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "zh", 20, 20) - #nostr_client_test_image("a beautiful purple ostrich watching the sunset") + nostr_client_test_image("a beautiful purple ostrich watching the sunset") #cashutoken = "cashuAeyJ0b2tlbiI6W3sicHJvb2ZzIjpbeyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MSwiQyI6IjAyNWU3ODZhOGFkMmExYTg0N2YxMzNiNGRhM2VhMGIyYWRhZGFkOTRiYzA4M2E2NWJjYjFlOTgwYTE1NGIyMDA2NCIsInNlY3JldCI6InQ1WnphMTZKMGY4UElQZ2FKTEg4V3pPck5rUjhESWhGa291LzVzZFd4S0U9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6NCwiQyI6IjAyOTQxNmZmMTY2MzU5ZWY5ZDc3MDc2MGNjZmY0YzliNTMzMzVmZTA2ZGI5YjBiZDg2Njg5Y2ZiZTIzMjVhYWUwYiIsInNlY3JldCI6IlRPNHB5WE43WlZqaFRQbnBkQ1BldWhncm44UHdUdE5WRUNYWk9MTzZtQXM9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MTYsIkMiOiIwMmRiZTA3ZjgwYmMzNzE0N2YyMDJkNTZiMGI3ZTIzZTdiNWNkYTBhNmI3Yjg3NDExZWYyOGRiZDg2NjAzNzBlMWIiLCJzZWNyZXQiOiJHYUNIdHhzeG9HM3J2WWNCc0N3V0YxbU1NVXczK0dDN1RKRnVwOHg1cURzPSJ9XSwibWludCI6Imh0dHBzOi8vbG5iaXRzLmJpdGNvaW5maXhlc3RoaXMub3JnL2Nhc2h1L2FwaS92MS9ScDlXZGdKZjlxck51a3M1eVQ2SG5rIn1dfQ==" - #nostr_client_test_image_private("a beautiful ostrich watching the sunset", cashutoken ) + #nostr_client_test_image_private("a beautiful ostrich watching the sunset") class NotificationHandler(HandleNotification): def handle(self, relay_url, event): print(f"Received new event from {relay_url}: {event.as_json()}") From f2c7aa3c1e32d06755c80dc4198550a9a89aff4c Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sun, 17 Dec 2023 14:38:58 +0100 Subject: [PATCH 02/16] option to use venv or not per module, added tts --- .gitignore | 1 + main.py | 7 +- nostr_dvm/dvm.py | 28 ++-- nostr_dvm/interfaces/dvmtaskinterface.py | 39 ++--- nostr_dvm/tasks/advanced_search.py | 24 +--- nostr_dvm/tasks/convert_media.py | 27 +--- nostr_dvm/tasks/discovery_inactive_follows.py | 21 +-- .../tasks/imagegeneration_openai_dalle.py | 22 +-- .../tasks/imagegeneration_replicate_sdxl.py | 19 +-- nostr_dvm/tasks/textextraction_google.py | 17 +-- nostr_dvm/tasks/textextraction_pdf.py | 17 +-- nostr_dvm/tasks/textgeneration_llmlite.py | 18 +-- nostr_dvm/tasks/texttospeech.py | 136 ++++++++++++++++++ nostr_dvm/tasks/translation_google.py | 20 +-- nostr_dvm/tasks/translation_libretranslate.py | 17 +-- nostr_dvm/tasks/trending_notes_nostrband.py | 18 +-- .../tasks/videogeneration_replicate_svd.py | 19 +-- nostr_dvm/utils/definitions.py | 24 ++-- nostr_dvm/utils/dvmconfig.py | 3 +- requirements.txt | 107 -------------- tests/test_dvm_client.py | 27 +++- 21 files changed, 247 insertions(+), 364 deletions(-) create mode 100644 nostr_dvm/tasks/texttospeech.py delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 5bb30f3..c2338a8 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,4 @@ backends/nserver/venv backends/nserver/cache backends/nserver/modules/image_upscale/weights cache/venvs/ +cache/input.wav diff --git a/main.py b/main.py index 79355cf..facadab 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ import dotenv from nostr_dvm.bot import Bot from nostr_dvm.tasks import videogeneration_replicate_svd, imagegeneration_replicate_sdxl, textgeneration_llmlite, \ trending_notes_nostrband, discovery_inactive_follows, translation_google, textextraction_pdf, \ - translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle + translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle, texttospeech from nostr_dvm.utils.admin_utils import AdminConfig from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions @@ -133,6 +133,11 @@ def playground(): ollama = textgeneration_llmlite.build_example("LLM", "llmlite", admin_config) bot_config.SUPPORTED_DVMS.append(ollama) ollama.run() + + tts = texttospeech.build_example("Text To Speech Test", "tts", admin_config) + bot_config.SUPPORTED_DVMS.append(tts) + tts.run() + # Run the bot Bot(bot_config) # Keep the main function alive for libraries that require it, like openai diff --git a/nostr_dvm/dvm.py b/nostr_dvm/dvm.py index 27c09ca..32f899f 100644 --- a/nostr_dvm/dvm.py +++ b/nostr_dvm/dvm.py @@ -473,20 +473,24 @@ class DVM: if task == dvm.TASK: request_form = dvm.create_request_from_nostr_event(job_event, self.client, self.dvm_config) - python_bin = (r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] - + "/bin/python") - subprocess.run([python_bin, dvm_config.SCRIPT, - '--request', json.dumps(request_form), - '--identifier', dvm_config.IDENTIFIER, - '--output', 'output.txt']) - print("Finished processing, loading data..") + if dvm_config.USE_OWN_VENV: + python_bin = (r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] + + "/bin/python") + retcode = subprocess.call([python_bin, dvm_config.SCRIPT, + '--request', json.dumps(request_form), + '--identifier', dvm_config.IDENTIFIER, + '--output', 'output.txt']) + print("Finished processing, loading data..") - with open(os.path.abspath('output.txt')) as f: - result = f.readlines()[0] - print(result) - #f.close() - os.remove(os.path.abspath('output.txt')) + with open(os.path.abspath('output.txt')) as f: + result = f.readlines()[0] + print(result) + #f.close() + os.remove(os.path.abspath('output.txt')) + else: #Some components might have issues with running code in otuside venv. + # We install locally in these cases for now + result = dvm.process(request_form) try: post_processed = dvm.post_process(str(result), job_event) send_nostr_reply_event(post_processed, job_event.as_json()) diff --git a/nostr_dvm/interfaces/dvmtaskinterface.py b/nostr_dvm/interfaces/dvmtaskinterface.py index 83125d7..1d0d08d 100644 --- a/nostr_dvm/interfaces/dvmtaskinterface.py +++ b/nostr_dvm/interfaces/dvmtaskinterface.py @@ -9,7 +9,7 @@ from nostr_sdk import Keys from nostr_dvm.dvm import DVM from nostr_dvm.utils.admin_utils import AdminConfig from nostr_dvm.utils.dvmconfig import DVMConfig -from nostr_dvm.utils.nip89_utils import NIP89Config +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag from nostr_dvm.utils.output_utils import post_process_result @@ -28,13 +28,11 @@ class DVMTaskInterface: admin_config: AdminConfig dependencies = [] - def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None, task=None): self.init(name, dvm_config, admin_config, nip89config, task) self.options = options - self.make_venv(dvm_config) - + self.install_dependencies(dvm_config) def init(self, name, dvm_config, admin_config=None, nip89config=None, task=None): self.NAME = name @@ -58,15 +56,24 @@ class DVMTaskInterface: self.dvm_config = dvm_config self.admin_config = admin_config - def make_venv(self, dvm_config): + def install_dependencies(self, dvm_config): if dvm_config.SCRIPT != "": - dir = r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] - if not os.path.isdir(dir): - print(dir) - create(dir, with_pip=True, upgrade_deps=True) - for (module, package) in self.dependencies: - print("Installing Module: " + module) - run(["bin/pip", "install", package], cwd=dir) + if self.dvm_config.USE_OWN_VENV: + dir = r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] + if not os.path.isdir(dir): + print(dir) + create(dir, with_pip=True, upgrade_deps=True) + for (module, package) in self.dependencies: + print("Installing Venv Module: " + module) + run(["bin/pip", "install", package], cwd=dir) + else: + for module, package in self.dependencies: + if module != "nostr-dvm": + try: + __import__(module) + except ImportError: + print("Installing global Module: " + module) + subprocess.check_call([sys.executable, "-m", "pip", "install", package]) def run(self): nostr_dvm_thread = Thread(target=self.DVM, args=[self.dvm_config, self.admin_config]) @@ -120,10 +127,4 @@ class DVMTaskInterface: def write_output(result, output): with open(os.path.abspath(output), 'w') as f: f.write(result) - #f.close() - - def process_venv(self): - pass - - if __name__ == '__main__': - process_venv() + # f.close() diff --git a/nostr_dvm/tasks/advanced_search.py b/nostr_dvm/tasks/advanced_search.py index 7bd2816..217ea18 100644 --- a/nostr_dvm/tasks/advanced_search.py +++ b/nostr_dvm/tasks/advanced_search.py @@ -1,14 +1,10 @@ import json import os from datetime import timedelta -from pathlib import Path - -import dotenv from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig -from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag @@ -186,21 +182,5 @@ def process_venv(): result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# process_venv() - - - #if env_path.is_file(): - # print(f'loading environment from {env_path.resolve()}') - # dotenv.load_dotenv(env_path, verbose=True, override=True) - #else: - # raise FileNotFoundError(f'.env file not found at {env_path} ') - # - #admin_config = AdminConfig() - #admin_config.REBROADCAST_NIP89 = False - #admin_config.UPDATE_PROFILE = False - - #dvm = build_example("Advanced Nostr Search", "discovery_content_search", admin_config) - #dvm.run() - - #keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/convert_media.py b/nostr_dvm/tasks/convert_media.py index 20ff106..0de9ba6 100644 --- a/nostr_dvm/tasks/convert_media.py +++ b/nostr_dvm/tasks/convert_media.py @@ -1,12 +1,7 @@ import json import os -from pathlib import Path - -import dotenv - from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig -from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag @@ -102,8 +97,7 @@ def build_example(name, identifier, admin_config): } nip89config = NIP89Config() - nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) - nip89config.CONTENT = json.dumps(nip89info) + return MediaConverter(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config) @@ -115,21 +109,6 @@ def process_venv(): result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# process_venv() - #env_path = Path('.env') - #if env_path.is_file(): - # print(f'loading environment from {env_path.resolve()}') - # dotenv.load_dotenv(env_path, verbose=True, override=True) - #else: - # raise FileNotFoundError(f'.env file not found at {env_path} ') - - #admin_config = AdminConfig() - #admin_config.REBROADCAST_NIP89 = False - #admin_config.UPDATE_PROFILE = False - - #dvm = build_example("Media Bringer", "media_converter", admin_config) - #dvm.run() - - #keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/discovery_inactive_follows.py b/nostr_dvm/tasks/discovery_inactive_follows.py index 6a52df5..961380e 100644 --- a/nostr_dvm/tasks/discovery_inactive_follows.py +++ b/nostr_dvm/tasks/discovery_inactive_follows.py @@ -1,15 +1,12 @@ import json import os from datetime import timedelta -from pathlib import Path from threading import Thread -import dotenv from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig -from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag @@ -206,19 +203,5 @@ def process_venv(): result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# process_venv() - #env_path = Path('.env') - #if env_path.is_file(): - # print(f'loading environment from {env_path.resolve()}') - # dotenv.load_dotenv(env_path, verbose=True, override=True) - #else: - # raise FileNotFoundError(f'.env file not found at {env_path} ') - - #admin_config = AdminConfig() - #admin_config.REBROADCAST_NIP89 = False - #admin_config.UPDATE_PROFILE = False - #dvm = build_example("Bygones", "discovery_inactive_follows", admin_config) - #dvm.run() - - #keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/imagegeneration_openai_dalle.py b/nostr_dvm/tasks/imagegeneration_openai_dalle.py index 3412784..86ebdcb 100644 --- a/nostr_dvm/tasks/imagegeneration_openai_dalle.py +++ b/nostr_dvm/tasks/imagegeneration_openai_dalle.py @@ -2,15 +2,12 @@ import json import os import time from io import BytesIO -from pathlib import Path -import dotenv import requests from PIL import Image from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig -from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag @@ -129,6 +126,7 @@ class ImageGenerationDALLE(DVMTaskInterface): # playground or elsewhere def build_example(name, identifier, admin_config): dvm_config = build_default_config(identifier) + dvm_config.USE_OWN_VENV = True admin_config.LUD16 = dvm_config.LN_ADDRESS profit_in_sats = 10 cost_in_cent = 4.0 @@ -167,19 +165,5 @@ def process_venv(): DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# process_venv() - #env_path = Path('.env') - #if env_path.is_file(): - # print(f'loading environment from {env_path.resolve()}') - # dotenv.load_dotenv(env_path, verbose=True, override=True) - #else: - # raise FileNotFoundError(f'.env file not found at {env_path} ') - - #admin_config = AdminConfig() - #admin_config.REBROADCAST_NIP89 = False - #admin_config.UPDATE_PROFILE = False - #dvm = build_example("Dall-E 3", "dalle3", admin_config) - #dvm.run() - - #keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py b/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py index 0aa7ab0..5c3cdec 100644 --- a/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py +++ b/nostr_dvm/tasks/imagegeneration_replicate_sdxl.py @@ -6,6 +6,7 @@ from PIL import Image from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag @@ -152,19 +153,5 @@ def process_venv(): result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# process_venv() - #env_path = Path('.env') - #if env_path.is_file(): - # print(f'loading environment from {env_path.resolve()}') - # dotenv.load_dotenv(env_path, verbose=True, override=True) - #else: - # raise FileNotFoundError(f'.env file not found at {env_path} ') - - #admin_config = AdminConfig() - #admin_config.REBROADCAST_NIP89 = False - #admin_config.UPDATE_PROFILE = False - #dvm = build_example("Stable Diffusion XL", "replicate_sdxl", admin_config) - #dvm.run() - - #keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/textextraction_google.py b/nostr_dvm/tasks/textextraction_google.py index 2c24ef7..f5c6b69 100644 --- a/nostr_dvm/tasks/textextraction_google.py +++ b/nostr_dvm/tasks/textextraction_google.py @@ -166,18 +166,5 @@ def process_venv(): DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# env_path = Path('.env') -# if env_path.is_file(): -# print(f'loading environment from {env_path.resolve()}') -# dotenv.load_dotenv(env_path, verbose=True, override=True) -# else: -# raise FileNotFoundError(f'.env file not found at {env_path} ') -# -# admin_config = AdminConfig() -# admin_config.REBROADCAST_NIP89 = False -# admin_config.UPDATE_PROFILE = False -# dvm = build_example("Transcriptor", "speech_recognition", admin_config) -# dvm.run() - - # keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/textextraction_pdf.py b/nostr_dvm/tasks/textextraction_pdf.py index 8101840..d578299 100644 --- a/nostr_dvm/tasks/textextraction_pdf.py +++ b/nostr_dvm/tasks/textextraction_pdf.py @@ -126,18 +126,5 @@ def process_venv(): DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# env_path = Path('.env') -# if env_path.is_file(): -# print(f'loading environment from {env_path.resolve()}') -# dotenv.load_dotenv(env_path, verbose=True, override=True) -# else: -# raise FileNotFoundError(f'.env file not found at {env_path} ') - -# admin_config = AdminConfig() -# admin_config.REBROADCAST_NIP89 = False -# admin_config.UPDATE_PROFILE = False -# dvm = build_example("PDF Extractor", "pdf_extractor", admin_config) -# dvm.run() - -# keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/textgeneration_llmlite.py b/nostr_dvm/tasks/textgeneration_llmlite.py index 38090d7..9bee4c1 100644 --- a/nostr_dvm/tasks/textgeneration_llmlite.py +++ b/nostr_dvm/tasks/textgeneration_llmlite.py @@ -135,19 +135,5 @@ def process_venv(): DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# env_path = Path('.env') -# if env_path.is_file(): -# print(f'loading environment from {env_path.resolve()}') -# dotenv.load_dotenv(env_path, verbose=True, override=True) -# else: -# raise FileNotFoundError(f'.env file not found at {env_path} ') -# -# admin_config = AdminConfig() -# admin_config.REBROADCAST_NIP89 = False -# admin_config.UPDATE_PROFILE = False -# -# dvm = build_example("LLM", "llmlite", admin_config) -# dvm.run() - -# keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/texttospeech.py b/nostr_dvm/tasks/texttospeech.py new file mode 100644 index 0000000..1e99d1a --- /dev/null +++ b/nostr_dvm/tasks/texttospeech.py @@ -0,0 +1,136 @@ +import json +import os +os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" +from pathlib import Path + +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.output_utils import upload_media_to_hoster + +""" +This File contains a Module to generate Audio based on an input and a voice + +Accepted Inputs: Text +Outputs: Generated Audiofile +""" + + +class TextToSpeech(DVMTaskInterface): + KIND: int = EventDefinitions.KIND_NIP90_TEXT_TO_SPEECH + TASK: str = "text-to-speech" + FIX_COST: float = 0 + dependencies = [("nostr-dvm", "nostr-dvm"), + ("TTS", "TTS==0.22.0")] + + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, + admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) + super().__init__(name, dvm_config, nip89config, admin_config, options) + + def is_input_supported(self, tags): + for tag in tags: + if tag.as_vec()[0] == 'i': + input_value = tag.as_vec()[1] + input_type = tag.as_vec()[2] + if input_type != "text": + return False + + return True + + def create_request_from_nostr_event(self, event, client=None, dvm_config=None): + request_form = {"jobID": event.id().to_hex() + "_" + self.NAME.replace(" ", "")} + prompt = "test" + if self.options.get("input_file") and self.options.get("input_file") != "": + input_file = self.options['input_file'] + else: + input_file = "https://media.nostr.build/av/de104e3260be636533a56fd4468b905c1eb22b226143a997aa936b011122af8a.wav" + import urllib.request + if not Path.exists(Path(r'cache/input.wav')): + urllib.request.urlretrieve(input_file, "cache/input.wav") + language = "en" + + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + input_type = tag.as_vec()[2] + if input_type == "text": + prompt = tag.as_vec()[1] + if input_type == "url": + input_file = tag.as_vec()[1] + elif tag.as_vec()[0] == 'param': + param = tag.as_vec()[1] + if param == "language": # check for param type + language = tag.as_vec()[2] + + options = { + "prompt": prompt, + "input_wav": input_file, + "language": language + } + request_form['options'] = json.dumps(options) + + return request_form + + def process(self, request_form): + import torch + from TTS.api import TTS + options = DVMTaskInterface.set_options(request_form) + device = "cuda" if torch.cuda.is_available() else "cpu" + #else "mps" if torch.backends.mps.is_available() \ + + print(TTS().list_models()) + try: + tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device) + + tts.tts_to_file( + text=options["prompt"], + speaker_wav="cache/input.wav", language=options["language"], file_path="outputs/output.wav") + result = upload_media_to_hoster("outputs/output.wav") + return result + except Exception as e: + print("Error in Module: " + str(e)) + raise Exception(e) + + +# We build an example here that we can call by either calling this file directly from the main directory, +# or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the +# playground or elsewhere +def build_example(name, identifier, admin_config): + dvm_config = build_default_config(identifier) + admin_config.LUD16 = dvm_config.LN_ADDRESS + + options = {'input_file': ""} + + nip89info = { + "name": name, + "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", + "about": "I Generate Speech from Text", + "encryptionSupported": True, + "cashuAccepted": True, + "nip90Params": { + "language": { + "required": False, + "values": [] + } + } + } + + nip89config = NIP89Config() + nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) + nip89config.CONTENT = json.dumps(nip89info) + + return TextToSpeech(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, + options=options) + + +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = TextToSpeech(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) + +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/translation_google.py b/nostr_dvm/tasks/translation_google.py index e9bcee7..b1f17d8 100644 --- a/nostr_dvm/tasks/translation_google.py +++ b/nostr_dvm/tasks/translation_google.py @@ -154,21 +154,5 @@ def process_venv(): DVMTaskInterface.write_output(result, args.output) - -#if __name__ == '__main__': - #process_venv() - - #env_path = Path('.env') - #if env_path.is_file(): - # print(f'loading environment from {env_path.resolve()}') - # dotenv.load_dotenv(env_path, verbose=True, override=True) - #else: - # raise FileNotFoundError(f'.env file not found at {env_path} ') - - #admin_config = AdminConfig() - #admin_config.REBROADCAST_NIP89 = False - #admin_config.UPDATE_PROFILE = False - #dvm = build_example("Google Translator", "google_translator", admin_config) - #dvm.run() - - #keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/translation_libretranslate.py b/nostr_dvm/tasks/translation_libretranslate.py index 9a89c3b..c3d00c7 100644 --- a/nostr_dvm/tasks/translation_libretranslate.py +++ b/nostr_dvm/tasks/translation_libretranslate.py @@ -156,18 +156,5 @@ def process_venv(): result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) -# if __name__ == '__main__': -# env_path = Path('.env') -# if env_path.is_file(): -# print(f'loading environment from {env_path.resolve()}') -# dotenv.load_dotenv(env_path, verbose=True, override=True) -# else: -# raise FileNotFoundError(f'.env file not found at {env_path} ') -# -# admin_config = AdminConfig() -# admin_config.REBROADCAST_NIP89 = False -# admin_config.UPDATE_PROFILE = False -# dvm = build_example("Libre Translator", "libre_translator", admin_config) -# dvm.run() -# -# keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/trending_notes_nostrband.py b/nostr_dvm/tasks/trending_notes_nostrband.py index b915ed7..20c5cd5 100644 --- a/nostr_dvm/tasks/trending_notes_nostrband.py +++ b/nostr_dvm/tasks/trending_notes_nostrband.py @@ -125,19 +125,5 @@ def process_venv(): result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# env_path = Path('.env') -# if env_path.is_file(): -# print(f'loading environment from {env_path.resolve()}') -# dotenv.load_dotenv(env_path, verbose=True, override=True) -# else: -# raise FileNotFoundError(f'.env file not found at {env_path} ') -# -# admin_config = AdminConfig() -# admin_config.REBROADCAST_NIP89 = False -# admin_config.UPDATE_PROFILE = False -# -# dvm = build_example("Trending Notes on Nostr.band", "trending_notes_nostrband", admin_config) -# dvm.run() -# -# keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/tasks/videogeneration_replicate_svd.py b/nostr_dvm/tasks/videogeneration_replicate_svd.py index 7bf2fe8..f283c1c 100644 --- a/nostr_dvm/tasks/videogeneration_replicate_svd.py +++ b/nostr_dvm/tasks/videogeneration_replicate_svd.py @@ -37,7 +37,6 @@ class VideoGenerationReplicateSVD(DVMTaskInterface): def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_config: AdminConfig = None, options=None): dvm_config.SCRIPT = os.path.abspath(__file__) - print(dvm_config.SCRIPT) super().__init__(name, dvm_config, nip89config, admin_config, options) @@ -149,19 +148,5 @@ def process_venv(): DVMTaskInterface.write_output(result, args.output) -#if __name__ == '__main__': -# env_path = Path('.env') -# if env_path.is_file(): -# print(f'loading environment from {env_path.resolve()}') -# dotenv.load_dotenv(env_path, verbose=True, override=True) -# else: -# raise FileNotFoundError(f'.env file not found at {env_path} ') -# -# admin_config = AdminConfig() -# admin_config.REBROADCAST_NIP89 = False -# admin_config.UPDATE_PROFILE = False -# -# dvm = build_example("Stable Video Diffusion", "replicate_svd", admin_config) -# dvm.run() -# -# keep_alive() +if __name__ == '__main__': + process_venv() \ No newline at end of file diff --git a/nostr_dvm/utils/definitions.py b/nostr_dvm/utils/definitions.py index 6c3123e..5191ff0 100644 --- a/nostr_dvm/utils/definitions.py +++ b/nostr_dvm/utils/definitions.py @@ -9,27 +9,29 @@ class EventDefinitions: KIND_NIP94_METADATA = 1063 KIND_FEEDBACK = 7000 KIND_NIP90_EXTRACT_TEXT = 5000 - KIND_NIP90_RESULT_EXTRACT_TEXT = 6000 + KIND_NIP90_RESULT_EXTRACT_TEXT = KIND_NIP90_EXTRACT_TEXT + 1000 KIND_NIP90_SUMMARIZE_TEXT = 5001 - KIND_NIP90_RESULT_SUMMARIZE_TEXT = 6001 + KIND_NIP90_RESULT_SUMMARIZE_TEXT = KIND_NIP90_SUMMARIZE_TEXT + 1000 KIND_NIP90_TRANSLATE_TEXT = 5002 - KIND_NIP90_RESULT_TRANSLATE_TEXT = 6002 + KIND_NIP90_RESULT_TRANSLATE_TEXT = KIND_NIP90_TRANSLATE_TEXT + 1000 + KIND_NIP90_TEXT_TO_SPEECH = 5005 + KIND_NIP90_TEXT_TO_SPEECH_RESULT = KIND_NIP90_TEXT_TO_SPEECH + 1000 KIND_NIP90_GENERATE_TEXT = 5050 - KIND_NIP90_RESULT_GENERATE_TEXT = 6050 + KIND_NIP90_RESULT_GENERATE_TEXT = KIND_NIP90_GENERATE_TEXT + 1000 KIND_NIP90_GENERATE_IMAGE = 5100 - KIND_NIP90_RESULT_GENERATE_IMAGE = 6100 + KIND_NIP90_RESULT_GENERATE_IMAGE = KIND_NIP90_GENERATE_IMAGE + 1000 KIND_NIP90_CONVERT_VIDEO = 5200 - KIND_NIP90_RESULT_CONVERT_VIDEO = 6200 + KIND_NIP90_RESULT_CONVERT_VIDEO = KIND_NIP90_CONVERT_VIDEO + 1000 KIND_NIP90_GENERATE_VIDEO = 5202 - KIND_NIP90_RESULT_GENERATE_VIDEO = 6202 + KIND_NIP90_RESULT_GENERATE_VIDEO = KIND_NIP90_GENERATE_VIDEO + 1000 KIND_NIP90_CONTENT_DISCOVERY = 5300 - KIND_NIP90_RESULT_CONTENT_DISCOVERY = 6300 + KIND_NIP90_RESULT_CONTENT_DISCOVERY = KIND_NIP90_CONTENT_DISCOVERY + 1000 KIND_NIP90_PEOPLE_DISCOVERY = 5301 - KIND_NIP90_RESULT_PEOPLE_DISCOVERY = 6301 + KIND_NIP90_RESULT_PEOPLE_DISCOVERY = KIND_NIP90_PEOPLE_DISCOVERY + 1000 KIND_NIP90_CONTENT_SEARCH = 5302 - KIND_NIP90_RESULTS_CONTENT_SEARCH = 6302 + KIND_NIP90_RESULTS_CONTENT_SEARCH = KIND_NIP90_CONTENT_SEARCH + 1000 KIND_NIP90_GENERIC = 5999 - KIND_NIP90_RESULT_GENERIC = 6999 + KIND_NIP90_RESULT_GENERIC = KIND_NIP90_GENERIC + 1000 ANY_RESULT = [KIND_NIP90_RESULT_EXTRACT_TEXT, KIND_NIP90_RESULT_SUMMARIZE_TEXT, KIND_NIP90_RESULT_TRANSLATE_TEXT, diff --git a/nostr_dvm/utils/dvmconfig.py b/nostr_dvm/utils/dvmconfig.py index 976cd74..38d3c59 100644 --- a/nostr_dvm/utils/dvmconfig.py +++ b/nostr_dvm/utils/dvmconfig.py @@ -23,12 +23,13 @@ class DVMConfig: RELAY_TIMEOUT = 3 EXTERNAL_POST_PROCESS_TYPE = PostProcessFunctionType.NONE # Leave this on None, except the DVM is external - LNBITS_INVOICE_KEY = '' + LNBITS_INVOICE_KEY = '' # Will all automatically generated by default, or read from .env LNBITS_ADMIN_KEY = '' # In order to pay invoices, e.g. from the bot to DVMs, or reimburse users. LNBITS_URL = 'https://lnbits.com' LN_ADDRESS = '' SCRIPT = '' IDENTIFIER = '' + USE_OWN_VENV = True # Make an own venv for each dvm's process function.Disable if you want to install packages into main venv. Only recommended if you dont want to run dvms with different dependency versions DB: str NEW_USER_BALANCE: int = 0 # Free credits for new users NIP89: NIP89Config diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 867beb6..0000000 --- a/requirements.txt +++ /dev/null @@ -1,107 +0,0 @@ -aiohttp==3.9.1 -aiosignal==1.3.1 -anyio==3.7.1 -appdirs==1.4.4 -asn1crypto==1.5.1 -async-timeout==4.0.3 -attrs==23.1.0 -base58==2.1.1 -beautifulsoup4==4.12.2 -bech32==1.2.0 -bip32==3.4 -bitarray==2.8.3 -bitstring==3.1.9 -blessed==1.20.0 -cassidy==0.1.4 -certifi==2023.7.22 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -coincurve==18.0.0 -cryptography==41.0.4 -decorator==4.4.2 -distro==1.8.0 -ecdsa==0.18.0 -emoji==2.8.0 -enumb==0.1.5 -environs==9.5.0 -eva-decord==0.6.1 -exceptiongroup==1.1.3 -expo==0.1.2 -fastapi==0.103.0 -ffmpegio==0.8.5 -ffmpegio-core==0.8.5 -filelock==3.13.1 -frozenlist==1.4.0 -fsspec==2023.12.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.1 -huggingface-hub==0.19.4 -idna==3.4 -imageio==2.33.0 -imageio-ffmpeg==0.4.9 -importlib-metadata==6.8.0 -inquirer==3.1.3 -install==1.3.5 -instaloader==4.10.1 -Jinja2==3.1.2 -litellm==1.12.3 -lnurl==0.4.1 -loguru==0.7.2 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -mediatype==0.1.6 -mnemonic==0.20 -moviepy==2.0.0.dev2 -multidict==6.0.4 -nostr-sdk==0.0.5 -numpy==1.26.2 -openai==1.3.5 -outcome==1.2.0 -packaging==23.2 -pandas==2.1.3 -Pillow==10.1.0 -pluggy==1.3.0 -proglog==0.1.10 -pycparser==2.21 -pycryptodome==3.19.0 -pycryptodomex==3.19.0 -pydantic==1.10.13 -pydantic_core==2.14.5 -pypdf==3.17.1 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -python-editor==1.0.4 -pytube==15.0.0 -pytz==2023.3.post1 -PyUpload==0.1.4 -pyuseragents==1.0.5 -PyYAML==6.0.1 -readchar==4.0.5 -regex==2023.10.3 -replicate==0.21.1 -Represent==1.6.0.post0 -requests==2.31.0 -requests-toolbelt==1.0.0 -safeIO==1.2 -six==1.16.0 -sniffio==1.3.0 -socksio==1.0.0 -soupsieve==2.5 -SpeechRecognition==3.10.0 -SQLAlchemy==1.3.24 -sqlalchemy-aio==0.17.0 -starlette==0.27.0 -tiktoken==0.5.2 -tokenizers==0.15.0 -tqdm==4.66.1 -translatepy==2.3 -typing_extensions==4.8.0 -tzdata==2023.3 -urllib3==2.1.0 -uvicorn==0.23.2 -wcwidth==0.2.10 -websocket-client==1.6.4 -yarl==1.9.4 -zipp==3.17.0 \ No newline at end of file diff --git a/tests/test_dvm_client.py b/tests/test_dvm_client.py index 7a5dd35..c0e252f 100644 --- a/tests/test_dvm_client.py +++ b/tests/test_dvm_client.py @@ -65,6 +65,30 @@ def nostr_client_test_image(prompt): config = DVMConfig send_event(event, client=client, dvm_config=config) return event.as_json() +def nostr_client_test_tts(prompt): + keys = Keys.from_sk_str(check_and_set_private_key("test_client")) + + iTag = Tag.parse(["i", prompt, "text"]) + paramTag1 = Tag.parse(["param", "language", "en"]) + + + bidTag = Tag.parse(['bid', str(1000 * 1000), str(1000 * 1000)]) + relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"]) + alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to generate TTSt"]) + event = EventBuilder(EventDefinitions.KIND_NIP90_TEXT_TO_SPEECH, str("Generate an Audio File."), + [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys) + + relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"] + + client = Client(keys) + for relay in relay_list: + client.add_relay(relay) + client.connect() + config = DVMConfig + send_event(event, client=client, dvm_config=config) + return event.as_json() def nostr_client_test_image_private(prompt, cashutoken): @@ -127,8 +151,9 @@ def nostr_client(): #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) #nostr_client_test_translation("note1p8cx2dz5ss5gnk7c59zjydcncx6a754c0hsyakjvnw8xwlm5hymsnc23rs", "event", "es", 20,20) #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "zh", 20, 20) + #nostr_client_test_image("a beautiful purple ostrich watching the sunset") + nostr_client_test_tts("Hello, this is a test. One two three.") - nostr_client_test_image("a beautiful purple ostrich watching the sunset") #cashutoken = "cashuAeyJ0b2tlbiI6W3sicHJvb2ZzIjpbeyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MSwiQyI6IjAyNWU3ODZhOGFkMmExYTg0N2YxMzNiNGRhM2VhMGIyYWRhZGFkOTRiYzA4M2E2NWJjYjFlOTgwYTE1NGIyMDA2NCIsInNlY3JldCI6InQ1WnphMTZKMGY4UElQZ2FKTEg4V3pPck5rUjhESWhGa291LzVzZFd4S0U9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6NCwiQyI6IjAyOTQxNmZmMTY2MzU5ZWY5ZDc3MDc2MGNjZmY0YzliNTMzMzVmZTA2ZGI5YjBiZDg2Njg5Y2ZiZTIzMjVhYWUwYiIsInNlY3JldCI6IlRPNHB5WE43WlZqaFRQbnBkQ1BldWhncm44UHdUdE5WRUNYWk9MTzZtQXM9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MTYsIkMiOiIwMmRiZTA3ZjgwYmMzNzE0N2YyMDJkNTZiMGI3ZTIzZTdiNWNkYTBhNmI3Yjg3NDExZWYyOGRiZDg2NjAzNzBlMWIiLCJzZWNyZXQiOiJHYUNIdHhzeG9HM3J2WWNCc0N3V0YxbU1NVXczK0dDN1RKRnVwOHg1cURzPSJ9XSwibWludCI6Imh0dHBzOi8vbG5iaXRzLmJpdGNvaW5maXhlc3RoaXMub3JnL2Nhc2h1L2FwaS92MS9ScDlXZGdKZjlxck51a3M1eVQ2SG5rIn1dfQ==" #nostr_client_test_image_private("a beautiful ostrich watching the sunset") From e6e9eca9ce15e3e008a749d1657e900ea86b8790 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sun, 17 Dec 2023 14:48:16 +0100 Subject: [PATCH 03/16] update version --- .gitignore | 1 + requirements.txt | 107 ----------------------------------------------- setup.py | 2 +- 3 files changed, 2 insertions(+), 108 deletions(-) delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 54c9476..0881ab7 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,4 @@ db/* backends/nserver/venv backends/nserver/cache backends/nserver/modules/image_upscale/weights +cache/ diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 867beb6..0000000 --- a/requirements.txt +++ /dev/null @@ -1,107 +0,0 @@ -aiohttp==3.9.1 -aiosignal==1.3.1 -anyio==3.7.1 -appdirs==1.4.4 -asn1crypto==1.5.1 -async-timeout==4.0.3 -attrs==23.1.0 -base58==2.1.1 -beautifulsoup4==4.12.2 -bech32==1.2.0 -bip32==3.4 -bitarray==2.8.3 -bitstring==3.1.9 -blessed==1.20.0 -cassidy==0.1.4 -certifi==2023.7.22 -cffi==1.16.0 -charset-normalizer==3.3.2 -click==8.1.7 -coincurve==18.0.0 -cryptography==41.0.4 -decorator==4.4.2 -distro==1.8.0 -ecdsa==0.18.0 -emoji==2.8.0 -enumb==0.1.5 -environs==9.5.0 -eva-decord==0.6.1 -exceptiongroup==1.1.3 -expo==0.1.2 -fastapi==0.103.0 -ffmpegio==0.8.5 -ffmpegio-core==0.8.5 -filelock==3.13.1 -frozenlist==1.4.0 -fsspec==2023.12.1 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.1 -huggingface-hub==0.19.4 -idna==3.4 -imageio==2.33.0 -imageio-ffmpeg==0.4.9 -importlib-metadata==6.8.0 -inquirer==3.1.3 -install==1.3.5 -instaloader==4.10.1 -Jinja2==3.1.2 -litellm==1.12.3 -lnurl==0.4.1 -loguru==0.7.2 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -mediatype==0.1.6 -mnemonic==0.20 -moviepy==2.0.0.dev2 -multidict==6.0.4 -nostr-sdk==0.0.5 -numpy==1.26.2 -openai==1.3.5 -outcome==1.2.0 -packaging==23.2 -pandas==2.1.3 -Pillow==10.1.0 -pluggy==1.3.0 -proglog==0.1.10 -pycparser==2.21 -pycryptodome==3.19.0 -pycryptodomex==3.19.0 -pydantic==1.10.13 -pydantic_core==2.14.5 -pypdf==3.17.1 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -python-editor==1.0.4 -pytube==15.0.0 -pytz==2023.3.post1 -PyUpload==0.1.4 -pyuseragents==1.0.5 -PyYAML==6.0.1 -readchar==4.0.5 -regex==2023.10.3 -replicate==0.21.1 -Represent==1.6.0.post0 -requests==2.31.0 -requests-toolbelt==1.0.0 -safeIO==1.2 -six==1.16.0 -sniffio==1.3.0 -socksio==1.0.0 -soupsieve==2.5 -SpeechRecognition==3.10.0 -SQLAlchemy==1.3.24 -sqlalchemy-aio==0.17.0 -starlette==0.27.0 -tiktoken==0.5.2 -tokenizers==0.15.0 -tqdm==4.66.1 -translatepy==2.3 -typing_extensions==4.8.0 -tzdata==2023.3 -urllib3==2.1.0 -uvicorn==0.23.2 -wcwidth==0.2.10 -websocket-client==1.6.4 -yarl==1.9.4 -zipp==3.17.0 \ No newline at end of file diff --git a/setup.py b/setup.py index fcf9d09..5dcb524 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.2' +VERSION = '0.0.3' DESCRIPTION = 'A framework to build and run NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') From 62ccbc09f7785fa80441a058af50d702c36b0480 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sun, 17 Dec 2023 15:43:47 +0100 Subject: [PATCH 04/16] fix setup --- setup.py | 2 +- test.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 test.py diff --git a/setup.py b/setup.py index 5dcb524..ffa36b6 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( author_email="believethehypeonnostr@proton.me", description=DESCRIPTION, long_description=LONG_DESCRIPTION, - packages=find_packages(include=['nostr_dvm']), + packages=find_packages(include=['nostr_dvm', 'nostr_dvm.backends', 'nostr_dvm.interfaces', 'nostr_dvm.tasks', 'nostr_dvm.utils']), install_requires=["nostr-sdk==0.0.5", "bech32==1.2.0", "pycryptodome==3.19.0", diff --git a/test.py b/test.py new file mode 100644 index 0000000..b50f471 --- /dev/null +++ b/test.py @@ -0,0 +1,44 @@ +import json + +import nostr_dvm +from nostr_dvm.tasks.texttospeech import TextToSpeech +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.dvmconfig import build_default_config +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag + +identifier = "tts" +name = "Guy Swann Clone" + +dvm_config = build_default_config(identifier) +admin_config = AdminConfig() +admin_config.REBROADCAST_NIP89 = False +admin_config.UPDATE_PROFILE = False +admin_config.LUD16 = dvm_config.LN_ADDRESS + +#Use default file if paramter is empty, else overwrite with any local wav file +options = {'input_file': ""} + +nip89info = { + "name": name, + "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", + "about": "I Generate Speech from Text", + "encryptionSupported": True, + "cashuAccepted": True, + "nip90Params": { + "language": { + "required": False, + "values": [] + } + } +} + +nip89config = NIP89Config() +nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) +nip89config.CONTENT = json.dumps(nip89info) + +tts = TextToSpeech(name=name, + dvm_config=dvm_config, + nip89config=nip89config, + admin_config=admin_config, + options=options) +tts.run() \ No newline at end of file From 0b5a3b69494a4365a66d36ee02ac3bf2827d2a01 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sun, 17 Dec 2023 15:44:09 +0100 Subject: [PATCH 05/16] Delete test.py --- test.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index b50f471..0000000 --- a/test.py +++ /dev/null @@ -1,44 +0,0 @@ -import json - -import nostr_dvm -from nostr_dvm.tasks.texttospeech import TextToSpeech -from nostr_dvm.utils.admin_utils import AdminConfig -from nostr_dvm.utils.dvmconfig import build_default_config -from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag - -identifier = "tts" -name = "Guy Swann Clone" - -dvm_config = build_default_config(identifier) -admin_config = AdminConfig() -admin_config.REBROADCAST_NIP89 = False -admin_config.UPDATE_PROFILE = False -admin_config.LUD16 = dvm_config.LN_ADDRESS - -#Use default file if paramter is empty, else overwrite with any local wav file -options = {'input_file': ""} - -nip89info = { - "name": name, - "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", - "about": "I Generate Speech from Text", - "encryptionSupported": True, - "cashuAccepted": True, - "nip90Params": { - "language": { - "required": False, - "values": [] - } - } -} - -nip89config = NIP89Config() -nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) -nip89config.CONTENT = json.dumps(nip89info) - -tts = TextToSpeech(name=name, - dvm_config=dvm_config, - nip89config=nip89config, - admin_config=admin_config, - options=options) -tts.run() \ No newline at end of file From 2adc87d925add46071fbe2051a430a7c0c719db4 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sun, 17 Dec 2023 15:44:53 +0100 Subject: [PATCH 06/16] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ffa36b6..9d40f50 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.3' +VERSION = '0.0.4' DESCRIPTION = 'A framework to build and run NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') From 79deb2495ca353c29969787b54652f0dc22f8b87 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Sun, 17 Dec 2023 22:28:05 +0100 Subject: [PATCH 07/16] fixes related to creating folders --- nostr_dvm/tasks/texttospeech.py | 14 ++++++++------ nostr_dvm/utils/database_utils.py | 6 ++++-- setup.py | 2 +- tests/test_dvm_client.py | 4 ++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/nostr_dvm/tasks/texttospeech.py b/nostr_dvm/tasks/texttospeech.py index 1e99d1a..eca7523 100644 --- a/nostr_dvm/tasks/texttospeech.py +++ b/nostr_dvm/tasks/texttospeech.py @@ -2,6 +2,7 @@ import json import os os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" from pathlib import Path +import urllib.request from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig @@ -21,7 +22,7 @@ Outputs: Generated Audiofile class TextToSpeech(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_TEXT_TO_SPEECH TASK: str = "text-to-speech" - FIX_COST: float = 0 + FIX_COST: float = 200 dependencies = [("nostr-dvm", "nostr-dvm"), ("TTS", "TTS==0.22.0")] @@ -46,10 +47,10 @@ class TextToSpeech(DVMTaskInterface): if self.options.get("input_file") and self.options.get("input_file") != "": input_file = self.options['input_file'] else: - input_file = "https://media.nostr.build/av/de104e3260be636533a56fd4468b905c1eb22b226143a997aa936b011122af8a.wav" - import urllib.request - if not Path.exists(Path(r'cache/input.wav')): - urllib.request.urlretrieve(input_file, "cache/input.wav") + if not Path.exists(Path('cache/input.wav')): + input_file_url = "https://media.nostr.build/av/de104e3260be636533a56fd4468b905c1eb22b226143a997aa936b011122af8a.wav" + urllib.request.urlretrieve(input_file_url, "cache/input.wav") + input_file = "cache/input.wav" language = "en" for tag in event.tags(): @@ -86,7 +87,7 @@ class TextToSpeech(DVMTaskInterface): tts.tts_to_file( text=options["prompt"], - speaker_wav="cache/input.wav", language=options["language"], file_path="outputs/output.wav") + speaker_wav=options["input_wav"], language=options["language"], file_path="outputs/output.wav") result = upload_media_to_hoster("outputs/output.wav") return result except Exception as e: @@ -101,6 +102,7 @@ def build_example(name, identifier, admin_config): dvm_config = build_default_config(identifier) admin_config.LUD16 = dvm_config.LN_ADDRESS + #use an alternative local wav file you want to use for cloning options = {'input_file': ""} nip89info = { diff --git a/nostr_dvm/utils/database_utils.py b/nostr_dvm/utils/database_utils.py index 32e44a1..5180f36 100644 --- a/nostr_dvm/utils/database_utils.py +++ b/nostr_dvm/utils/database_utils.py @@ -26,8 +26,10 @@ class User: def create_sql_table(db): try: import os - if not os.path.exists(r'.\db'): - os.makedirs(r'.\db') + if not os.path.exists(r'db'): + os.makedirs(r'db') + if not os.path.exists(r'outputs'): + os.makedirs(r'outputs') con = sqlite3.connect(db) cur = con.cursor() cur.execute(""" CREATE TABLE IF NOT EXISTS users ( diff --git a/setup.py b/setup.py index 9d40f50..529b5d2 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.4' +VERSION = '0.0.5' DESCRIPTION = 'A framework to build and run NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') diff --git a/tests/test_dvm_client.py b/tests/test_dvm_client.py index c0e252f..b47d414 100644 --- a/tests/test_dvm_client.py +++ b/tests/test_dvm_client.py @@ -152,7 +152,7 @@ def nostr_client(): #nostr_client_test_translation("note1p8cx2dz5ss5gnk7c59zjydcncx6a754c0hsyakjvnw8xwlm5hymsnc23rs", "event", "es", 20,20) #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "zh", 20, 20) #nostr_client_test_image("a beautiful purple ostrich watching the sunset") - nostr_client_test_tts("Hello, this is a test. One two three.") + nostr_client_test_tts("Hello, this is a test. Mic check one, two.") #cashutoken = "cashuAeyJ0b2tlbiI6W3sicHJvb2ZzIjpbeyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MSwiQyI6IjAyNWU3ODZhOGFkMmExYTg0N2YxMzNiNGRhM2VhMGIyYWRhZGFkOTRiYzA4M2E2NWJjYjFlOTgwYTE1NGIyMDA2NCIsInNlY3JldCI6InQ1WnphMTZKMGY4UElQZ2FKTEg4V3pPck5rUjhESWhGa291LzVzZFd4S0U9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6NCwiQyI6IjAyOTQxNmZmMTY2MzU5ZWY5ZDc3MDc2MGNjZmY0YzliNTMzMzVmZTA2ZGI5YjBiZDg2Njg5Y2ZiZTIzMjVhYWUwYiIsInNlY3JldCI6IlRPNHB5WE43WlZqaFRQbnBkQ1BldWhncm44UHdUdE5WRUNYWk9MTzZtQXM9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MTYsIkMiOiIwMmRiZTA3ZjgwYmMzNzE0N2YyMDJkNTZiMGI3ZTIzZTdiNWNkYTBhNmI3Yjg3NDExZWYyOGRiZDg2NjAzNzBlMWIiLCJzZWNyZXQiOiJHYUNIdHhzeG9HM3J2WWNCc0N3V0YxbU1NVXczK0dDN1RKRnVwOHg1cURzPSJ9XSwibWludCI6Imh0dHBzOi8vbG5iaXRzLmJpdGNvaW5maXhlc3RoaXMub3JnL2Nhc2h1L2FwaS92MS9ScDlXZGdKZjlxck51a3M1eVQ2SG5rIn1dfQ==" @@ -162,7 +162,7 @@ def nostr_client(): print(f"Received new event from {relay_url}: {event.as_json()}") if event.kind() == 7000: print("[Nostr Client]: " + event.as_json()) - elif event.kind() > 6000 and event.kind() < 6999: + elif 6000 < event.kind() < 6999: print("[Nostr Client]: " + event.as_json()) print("[Nostr Client]: " + event.content()) From 14637c0b507e3b3a4d7ebff18fbbec1a4ab3ad15 Mon Sep 17 00:00:00 2001 From: believethehype Date: Sun, 17 Dec 2023 22:32:38 +0100 Subject: [PATCH 08/16] Update README.md --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8e90e7e..672ff2f 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ This means the project is in alpha status, interfaces might still change/break a Create a new venv by running `"python -m venv venv"` - Place .env file (based on .env_example) in main folder. - - the framework will automatically create keys for your dvms in this file. - - Install requirements.txt - - Run python main.py. + - Create a `LNbits` account on an accessible instance of your choice, enter one account's id and admin key (this account will create other accounts for the dvms) + - the framework will then automatically create keys, nip89 tags and zapable NIP57 `lightning addresses` for your dvms in this file. + - pip install nostr-dvm + - Run python3 main.py. In each task component DVM examples are already prepared. Feel free to play along with the existing ones. You can also add new tasks by using the interface, just like the existing tasks in the `tasks` folder. @@ -21,8 +22,6 @@ You can also add new tasks by using the interface, just like the existing tasks A `bot` is running by default that lists and communicates with the `DVMs` added to it, so your DVMs can be controled via any regular social client as well. -The Framework optionally supports `LNbits` to create invoices instead of using a `lightning address`. If LNBits is not used, -make sure your nostr accounts have a valid lightning address. - +If LNBits is not used, make sure your DVM's nostr accounts have a valid lightning address. A tutorial on how to add additional tasks, as well as the larger server backend will be added at a later stage. From 1a8ab7382d93831ef8469af579d2adf81ef9341e Mon Sep 17 00:00:00 2001 From: Believethehype Date: Mon, 18 Dec 2023 09:27:01 +0100 Subject: [PATCH 09/16] fixes --- nostr_dvm/utils/mediasource_utils.py | 12 ++++++------ setup.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nostr_dvm/utils/mediasource_utils.py b/nostr_dvm/utils/mediasource_utils.py index 9b00c01..462620e 100644 --- a/nostr_dvm/utils/mediasource_utils.py +++ b/nostr_dvm/utils/mediasource_utils.py @@ -264,7 +264,7 @@ def get_youtube(input_value, start, end, audioonly=True): filename = downloadYouTube(input_value, filepath, audioonly) except Exception as e: - print("Youtube" + str(e)) + print("Youtube " + str(e)) return filename, start, end try: o = urlparse(input_value) @@ -332,30 +332,30 @@ def get_media_link(url) -> (str, str): def downloadOvercast(source_url, target_location): - from scrapper.media_scrapper import OvercastDownload + from nostr_dvm.utils.scrapper.media_scrapper import OvercastDownload result = OvercastDownload(source_url, target_location) return result def downloadTwitter(videourl, path): - from scrapper.media_scrapper import XitterDownload + from nostr_dvm.utils.scrapper.media_scrapper import XitterDownload result = XitterDownload(videourl, path + "x.mp4") return result def downloadTikTok(videourl, path): - from scrapper.media_scrapper import TiktokDownloadAll + from nostr_dvm.utils.scrapper.media_scrapper import TiktokDownloadAll result = TiktokDownloadAll([videourl], path) return result def downloadInstagram(videourl, path): - from scrapper.media_scrapper import InstagramDownload + from nostr_dvm.utils.scrapper.media_scrapper import InstagramDownload result = InstagramDownload(videourl, "insta", path) return result def downloadYouTube(link, path, audioonly=True): - from scrapper.media_scrapper import YouTubeDownload + from nostr_dvm.utils.scrapper.media_scrapper import YouTubeDownload result = YouTubeDownload(link, path, audio_only=audioonly) return result diff --git a/setup.py b/setup.py index 529b5d2..a96c4d8 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from setuptools import setup, find_packages VERSION = '0.0.5' -DESCRIPTION = 'A framework to build and run NIP90 Data Vending Machines' -LONG_DESCRIPTION = ('A framework to build and run NIP90 Data Vending Machines. ' +DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' +LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') # Setting up From ba3182631e57f51432294ddd791b9bedef37c5c8 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Mon, 18 Dec 2023 10:03:45 +0100 Subject: [PATCH 10/16] some adjustments in terms of installation package --- nostr_dvm/interfaces/dvmtaskinterface.py | 4 +++- nostr_dvm/tasks/convert_media.py | 2 +- setup.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nostr_dvm/interfaces/dvmtaskinterface.py b/nostr_dvm/interfaces/dvmtaskinterface.py index 1d0d08d..3035ef4 100644 --- a/nostr_dvm/interfaces/dvmtaskinterface.py +++ b/nostr_dvm/interfaces/dvmtaskinterface.py @@ -59,13 +59,15 @@ class DVMTaskInterface: def install_dependencies(self, dvm_config): if dvm_config.SCRIPT != "": if self.dvm_config.USE_OWN_VENV: + dir = r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0] if not os.path.isdir(dir): print(dir) create(dir, with_pip=True, upgrade_deps=True) + self.dependencies.append(("nostr-dvm", "nostr-dvm")) for (module, package) in self.dependencies: print("Installing Venv Module: " + module) - run(["bin/pip", "install", package], cwd=dir) + run(["bin/pip", "install", "--force-reinstall", package], cwd=dir) else: for module, package in self.dependencies: if module != "nostr-dvm": diff --git a/nostr_dvm/tasks/convert_media.py b/nostr_dvm/tasks/convert_media.py index 0de9ba6..2d30475 100644 --- a/nostr_dvm/tasks/convert_media.py +++ b/nostr_dvm/tasks/convert_media.py @@ -89,7 +89,7 @@ def build_example(name, identifier, admin_config): "encryptionSupported": True, "cashuAccepted": True, "nip90Params": { - "media_format": { + "format": { "required": False, "values": ["video/mp4", "audio/mp3"] } diff --git a/setup.py b/setup.py index a96c4d8..7a0e5eb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.5' +VERSION = '0.0.6' DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') @@ -13,7 +13,8 @@ setup( author_email="believethehypeonnostr@proton.me", description=DESCRIPTION, long_description=LONG_DESCRIPTION, - packages=find_packages(include=['nostr_dvm', 'nostr_dvm.backends', 'nostr_dvm.interfaces', 'nostr_dvm.tasks', 'nostr_dvm.utils']), + packages=find_packages(include=['nostr_dvm', 'nostr_dvm.backends', 'nostr_dvm.interfaces', 'nostr_dvm.tasks', + 'nostr_dvm.utils', 'nostr_dvm.utils.scrapper']), install_requires=["nostr-sdk==0.0.5", "bech32==1.2.0", "pycryptodome==3.19.0", From 39821ac42e2cc422ce3c40ebcf3108b67c911019 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Mon, 18 Dec 2023 11:20:53 +0100 Subject: [PATCH 11/16] more fixes related to deployment --- nostr_dvm/utils/mediasource_utils.py | 38 +++++++++++++--------------- nostr_dvm/utils/scrapper/__init__.py | 1 - setup.py | 6 +++-- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/nostr_dvm/utils/mediasource_utils.py b/nostr_dvm/utils/mediasource_utils.py index 462620e..9ae98f7 100644 --- a/nostr_dvm/utils/mediasource_utils.py +++ b/nostr_dvm/utils/mediasource_utils.py @@ -6,10 +6,12 @@ import ffmpegio from decord import AudioReader, cpu import requests from nostr_dvm.utils.nostr_utils import get_event_by_id +from nostr_dvm.utils.scrapper.media_scrapper import OvercastDownload, XitterDownload, TiktokDownloadAll, \ + InstagramDownload, YouTubeDownload def input_data_file_duration(event, dvm_config, client, start=0, end=0): - #print("[" + dvm_config.NIP89.NAME + "] Getting Duration of the Media file..") + # print("[" + dvm_config.NIP89.NAME + "] Getting Duration of the Media file..") input_value = "" input_type = "" for tag in event.tags(): @@ -18,7 +20,7 @@ def input_data_file_duration(event, dvm_config, client, start=0, end=0): input_type = tag.as_vec()[2] if input_type == "text": - #For now, ingore length of any text, just return 1. + # For now, ingore length of any text, just return 1. return 1 if input_type == "event": # NIP94 event @@ -52,7 +54,8 @@ def input_data_file_duration(event, dvm_config, client, start=0, end=0): return 1 -def organize_input_media_data(input_value, input_type, start, end, dvm_config, client, process=True, media_format="audio/mp3") -> str: +def organize_input_media_data(input_value, input_type, start, end, dvm_config, client, process=True, + media_format="audio/mp3") -> str: if input_type == "event": # NIP94 event evt = get_event_by_id(input_value, client=client, config=dvm_config) if evt is not None: @@ -209,7 +212,7 @@ def get_overcast(input_value, start, end): print("Found overcast.fm Link.. downloading") start_time = start end_time = end - downloadOvercast(input_value, filename) + download_overcast(input_value, filename) finaltag = str(input_value).replace("https://overcast.fm/", "").split('/') if start == 0.0: if len(finaltag) > 1: @@ -227,7 +230,7 @@ def get_overcast(input_value, start, end): def get_TikTok(input_value, start, end): filepath = os.path.abspath(os.curdir + r'/outputs/') try: - filename = downloadTikTok(input_value, filepath) + filename = download_tik_tok(input_value, filepath) print(filename) except Exception as e: print(e) @@ -238,7 +241,7 @@ def get_TikTok(input_value, start, end): def get_Instagram(input_value, start, end): filepath = os.path.abspath(os.curdir + r'/outputs/') try: - filename = downloadInstagram(input_value, filepath) + filename = download_instagram(input_value, filepath) print(filename) except Exception as e: print(e) @@ -250,7 +253,7 @@ def get_Twitter(input_value, start, end): filepath = os.path.abspath(os.curdir) + r'/outputs/' cleanlink = str(input_value).replace("twitter.com", "x.com") try: - filename = downloadTwitter(cleanlink, filepath) + filename = download_twitter(cleanlink, filepath) except Exception as e: print(e) return "", start, end @@ -259,9 +262,10 @@ def get_Twitter(input_value, start, end): def get_youtube(input_value, start, end, audioonly=True): filepath = os.path.abspath(os.curdir) + r'/outputs/' + print(filepath) filename = "" try: - filename = downloadYouTube(input_value, filepath, audioonly) + filename = download_youtube(input_value, filepath, audioonly) except Exception as e: print("Youtube " + str(e)) @@ -331,31 +335,25 @@ def get_media_link(url) -> (str, str): return None, None -def downloadOvercast(source_url, target_location): - from nostr_dvm.utils.scrapper.media_scrapper import OvercastDownload +def download_overcast(source_url, target_location): result = OvercastDownload(source_url, target_location) return result -def downloadTwitter(videourl, path): - from nostr_dvm.utils.scrapper.media_scrapper import XitterDownload +def download_twitter(videourl, path): result = XitterDownload(videourl, path + "x.mp4") return result -def downloadTikTok(videourl, path): - from nostr_dvm.utils.scrapper.media_scrapper import TiktokDownloadAll +def download_tik_tok(videourl, path): result = TiktokDownloadAll([videourl], path) return result -def downloadInstagram(videourl, path): - from nostr_dvm.utils.scrapper.media_scrapper import InstagramDownload +def download_instagram(videourl, path): result = InstagramDownload(videourl, "insta", path) return result -def downloadYouTube(link, path, audioonly=True): - from nostr_dvm.utils.scrapper.media_scrapper import YouTubeDownload - result = YouTubeDownload(link, path, audio_only=audioonly) - return result +def download_youtube(link, path, audioonly=True): + return YouTubeDownload(link, path, audio_only=audioonly) diff --git a/nostr_dvm/utils/scrapper/__init__.py b/nostr_dvm/utils/scrapper/__init__.py index 9a970eb..e69de29 100644 --- a/nostr_dvm/utils/scrapper/__init__.py +++ b/nostr_dvm/utils/scrapper/__init__.py @@ -1 +0,0 @@ -from dvm import DVM \ No newline at end of file diff --git a/setup.py b/setup.py index 7a0e5eb..2ad52c7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.6' +VERSION = '0.0.7' DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') @@ -30,7 +30,9 @@ setup( "instaloader==4.10.1", "pytube==15.0.0", "moviepy==2.0.0.dev2", - "zipp==3.17.0" + "zipp==3.17.0", + "urllib3==2.1.0", + "typing_extensions==4.8.0" ], keywords=['nostr', 'nip90', 'dvm', 'data vending machine'], url="https://github.com/believethehype/nostrdvm", From 3043c188f5a3c1524b7355e79f776a2154bfae8a Mon Sep 17 00:00:00 2001 From: Believethehype Date: Mon, 18 Dec 2023 13:57:53 +0100 Subject: [PATCH 12/16] fixing media converter tools --- nostr_dvm/tasks/convert_media.py | 2 +- nostr_dvm/utils/mediasource_utils.py | 2 +- nostr_dvm/utils/scrapper/media_scrapper.py | 64 ++++++++++++++++++---- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/nostr_dvm/tasks/convert_media.py b/nostr_dvm/tasks/convert_media.py index 2d30475..fa8655c 100644 --- a/nostr_dvm/tasks/convert_media.py +++ b/nostr_dvm/tasks/convert_media.py @@ -4,7 +4,7 @@ from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface from nostr_dvm.utils.admin_utils import AdminConfig from nostr_dvm.utils.definitions import EventDefinitions from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config -from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.nip89_utils import NIP89Config from nostr_dvm.utils.mediasource_utils import organize_input_media_data from nostr_dvm.utils.output_utils import upload_media_to_hoster diff --git a/nostr_dvm/utils/mediasource_utils.py b/nostr_dvm/utils/mediasource_utils.py index 9ae98f7..6c7d40f 100644 --- a/nostr_dvm/utils/mediasource_utils.py +++ b/nostr_dvm/utils/mediasource_utils.py @@ -20,7 +20,7 @@ def input_data_file_duration(event, dvm_config, client, start=0, end=0): input_type = tag.as_vec()[2] if input_type == "text": - # For now, ingore length of any text, just return 1. + # For now, ignore length of any text, just return 1. return 1 if input_type == "event": # NIP94 event diff --git a/nostr_dvm/utils/scrapper/media_scrapper.py b/nostr_dvm/utils/scrapper/media_scrapper.py index bff4807..16ba2e1 100644 --- a/nostr_dvm/utils/scrapper/media_scrapper.py +++ b/nostr_dvm/utils/scrapper/media_scrapper.py @@ -18,7 +18,15 @@ def XitterDownload(source_url, target_location): features, variables = request_details["features"], request_details["variables"] def get_tokens(tweet_url): - html = requests.get(tweet_url) + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0", + "Accept": "*/*", + "Accept-Language": "de,en-US;q=0.7,en;q=0.3", + "Accept-Encoding": "gzip, deflate, br", + "TE": "trailers", + } + + html = requests.get(tweet_url, headers=headers) assert ( html.status_code == 200 @@ -34,7 +42,6 @@ def XitterDownload(source_url, target_location): ), f"Failed to find main.js file. If you are using the correct Twitter URL this suggests a bug in the script. Please open a GitHub issue and copy and paste this message. Tweet url: {tweet_url}" mainjs_url = mainjs_url[0] - mainjs = requests.get(mainjs_url) assert ( @@ -80,9 +87,11 @@ def XitterDownload(source_url, target_location): variables["tweetId"] = tweet_id return f"https://twitter.com/i/api/graphql/0hWvDhmW8YQ-S_ib3azIrw/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(variables))}&features={urllib.parse.quote(json.dumps(features))}" + # return f"https://api.twitter.com/graphql/ncDeACNGIApPMaqGVuF_rw/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(variables))}&features={urllib.parse.quote(json.dumps(features))}" def get_tweet_details(tweet_url, guest_token, bearer_token): tweet_id = re.findall(r"(?<=status/)\d+", tweet_url) + assert ( tweet_id is not None and len(tweet_id) == 1 ), f"Could not parse tweet id from your url. Make sure you are using the correct url. If you are, then file a GitHub issue and copy and paste this message. Tweet url: {tweet_url}" @@ -172,7 +181,7 @@ def XitterDownload(source_url, target_location): pattern = ( r'"expanded_url"\s*:\s*"https://x\.com/[^/]+/status/' + sid - + '/[^"]+",\s*"id_str"\s*:\s*"\d+",' + + r'/[^"]+",\s*"id_str"\s*:\s*"\d+",' ) matches = re.findall(pattern, j) if len(matches) > 0: @@ -240,17 +249,49 @@ def XitterDownload(source_url, target_location): urls = [x["url"] for x in results.values()] urls += container_matches return urls - return [x["url"] for x in results.values()] + def extract_mp4_fmp4(j): + """ + Extract the URL of the MP4 video from the detailed information of the tweet. + Returns a list of URLs, tweet IDs, and resolution information (dictionary type) + and a list of tweet IDs as return values. + """ + + # Empty list to store tweet IDs + tweet_id_list = [] + mp4_info_dict_list = [] + amplitude_pattern = re.compile( + r"(https://video.twimg.com/amplify_video/(\d+)/vid/(avc1/)(\d+x\d+)/[^.]+.mp4\?tag=\d+)" + ) + ext_tw_pattern = re.compile( + r"(https://video.twimg.com/ext_tw_video/(\d+)/pu/vid/(avc1/)?(\d+x\d+)/[^.]+.mp4\?tag=\d+)" + ) + tweet_video_pattern = re.compile(r'https://video.twimg.com/tweet_video/[^"]+') + container_pattern = re.compile(r'https://video.twimg.com/[^"]*container=fmp4') + + matches = amplitude_pattern.findall(j) + matches += ext_tw_pattern.findall(j) + container_matches = container_pattern.findall(j) + tweet_video_url_list = tweet_video_pattern.findall(j) + + for match in matches: + url, tweet_id, _, resolution = match + tweet_id_list.append(int(tweet_id)) + mp4_info_dict_list.append({"resolution": resolution, "url": url}) + + tweet_id_list = list(dict.fromkeys(tweet_id_list)) + + if len(container_matches) > 0: + for url in container_matches: + mp4_info_dict_list.append({"url": url}) + + return tweet_id_list, mp4_info_dict_list, tweet_video_url_list + def download_parts(url, output_filename): resp = requests.get(url, stream=True) - - # container begins with / ends with fmp4 and has a resolution in it we want to capture pattern = re.compile(r"(/[^\n]*/(\d+x\d+)/[^\n]*container=fmp4)") - matches = pattern.findall(resp.text) - max_res = 0 max_res_url = None @@ -301,12 +342,9 @@ def XitterDownload(source_url, target_location): def repost_check(j, exclude_replies=True): try: - # This line extract the index of the first reply reply_index = j.index('"conversationthread-') except ValueError: - # If there are no replies we use the enrire response data length reply_index = len(j) - # We truncate the response data to exclude replies if exclude_replies: j = j[0:reply_index] @@ -360,6 +398,7 @@ def XitterDownload(source_url, target_location): bearer_token, guest_token = get_tokens(tweet_url) resp = get_tweet_details(tweet_url, guest_token, bearer_token) mp4s = extract_mp4s(resp.text, tweet_url, target_all_videos) + if target_all_videos: video_counter = 1 original_urls = repost_check(resp.text, exclude_replies=False) @@ -377,6 +416,7 @@ def XitterDownload(source_url, target_location): download_parts(mp4, output_file) else: + # use a stream to download the file r = requests.get(mp4, stream=True) with open(output_file, "wb") as f: for chunk in r.iter_content(chunk_size=1024): @@ -475,7 +515,7 @@ def TiktokDownloadAll(linkList, path) -> str: for i in linkList: try: data['url'] = i - result = TikTokDownload(cookies, headers, data, "tiktok", path) # str(linkList.index(i)) + result = TikTokDownload(cookies, headers, data, "tiktok", path) # str(linkList.index(i)) return result except IndexError: parseDict = getDict() From 70e684f0b16097c68d084cb154b6e4760a79d0bb Mon Sep 17 00:00:00 2001 From: Believethehype Date: Tue, 19 Dec 2023 11:36:25 +0100 Subject: [PATCH 13/16] error handling if no .env file is provided --- main.py | 4 +++ nostr_dvm/backends/nova_server/utils.py | 3 --- nostr_dvm/utils/zap_utils.py | 34 ++++++++++--------------- setup.py | 2 +- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/main.py b/main.py index facadab..6a2bf38 100644 --- a/main.py +++ b/main.py @@ -146,6 +146,10 @@ def playground(): if __name__ == '__main__': env_path = Path('.env') + if not env_path.is_file(): + with open('.env', 'w') as f: + print("Wrting new .env file") + f.write('') if env_path.is_file(): print(f'loading environment from {env_path.resolve()}') dotenv.load_dotenv(env_path, verbose=True, override=True) diff --git a/nostr_dvm/backends/nova_server/utils.py b/nostr_dvm/backends/nova_server/utils.py index 613032e..2796cc8 100644 --- a/nostr_dvm/backends/nova_server/utils.py +++ b/nostr_dvm/backends/nova_server/utils.py @@ -43,9 +43,6 @@ def send_file_to_server(filepath, address): return result - # headers = {'Content-type': 'application/x-www-form-urlencoded'} - - """ check_n_server_status(request_form, address) Function that requests the status of the current process with the jobID (we use the Nostr event as jobID). diff --git a/nostr_dvm/utils/zap_utils.py b/nostr_dvm/utils/zap_utils.py index ed85ced..c9d8680 100644 --- a/nostr_dvm/utils/zap_utils.py +++ b/nostr_dvm/utils/zap_utils.py @@ -9,12 +9,12 @@ from Crypto.Cipher import AES from Crypto.Util.Padding import pad from bech32 import bech32_decode, convertbits, bech32_encode from nostr_sdk import nostr_sdk, PublicKey, SecretKey, Event, EventBuilder, Tag, Keys + from nostr_dvm.utils.nostr_utils import get_event_by_id, check_and_decrypt_own_tags import lnurl from hashlib import sha256 import dotenv - # TODO tor connection to lnbits # proxies = { # 'http': 'socks5h://127.0.0.1:9050', @@ -23,6 +23,7 @@ import dotenv proxies = {} + def parse_zap_event_tags(zap_event, keys, name, client, config): zapped_event = None invoice_amount = 0 @@ -126,10 +127,11 @@ def create_bolt11_lud16(lud16, amount): except: return None + def create_lnbits_account(name): if os.getenv("LNBITS_ADMIN_ID") is None or os.getenv("LNBITS_ADMIN_ID") == "": print("No admin id set, no wallet created.") - return + return "","","","", "failed" data = { 'admin_id': os.getenv("LNBITS_ADMIN_ID"), 'wallet_name': name, @@ -144,9 +146,11 @@ def create_lnbits_account(name): walletjson = json.loads(r.text) print(walletjson) if walletjson.get("wallets"): - return walletjson['wallets'][0]['inkey'], walletjson['wallets'][0]['adminkey'], walletjson['wallets'][0]['id'], walletjson['id'], "success" + return walletjson['wallets'][0]['inkey'], walletjson['wallets'][0]['adminkey'], walletjson['wallets'][0][ + 'id'], walletjson['id'], "success" except: print("error creating wallet") + return "", "", "", "", "failed" def check_bolt11_ln_bits_is_paid(payment_hash: str, config): @@ -278,9 +282,6 @@ def zap(lud16: str, amount: int, content, zapped_event: Event, keys, dvm_config, return None - - - def get_price_per_sat(currency): import requests @@ -298,13 +299,11 @@ def get_price_per_sat(currency): return price_currency_per_sat - def make_ln_address_nostdress(identifier, npub, pin, nostdressdomain): - #env_path = Path('.env') - #if env_path.is_file(): + # env_path = Path('.env') + # if env_path.is_file(): # dotenv.load_dotenv(env_path, verbose=True, override=True) - print(os.getenv("LNBITS_INVOICE_KEY_" + identifier.upper())) data = { 'name': identifier, @@ -314,10 +313,9 @@ def make_ln_address_nostdress(identifier, npub, pin, nostdressdomain): 'key': os.getenv("LNBITS_INVOICE_KEY_" + identifier.upper()), 'pin': pin, 'npub': npub, - 'currentname': " " + 'currentname': " " } - try: url = "https://" + nostdressdomain + "/api/easy/" res = requests.post(url, data=data) @@ -330,8 +328,8 @@ def make_ln_address_nostdress(identifier, npub, pin, nostdressdomain): print(e) return "", "" -def check_and_set_ln_bits_keys(identifier, npub): +def check_and_set_ln_bits_keys(identifier, npub): if not os.getenv("LNBITS_INVOICE_KEY_" + identifier.upper()): invoicekey, adminkey, walletid, userid, success = create_lnbits_account(identifier) add_key_to_env_file("LNBITS_INVOICE_KEY_" + identifier.upper(), invoicekey) @@ -341,7 +339,7 @@ def check_and_set_ln_bits_keys(identifier, npub): lnaddress = "" pin = "" - if os.getenv("NOSTDRESS_DOMAIN"): + if os.getenv("NOSTDRESS_DOMAIN") and success != "failed": print(os.getenv("NOSTDRESS_DOMAIN")) lnaddress, pin = make_ln_address_nostdress(identifier, npub, " ", os.getenv("NOSTDRESS_DOMAIN")) add_key_to_env_file("LNADDRESS_" + identifier.upper(), lnaddress) @@ -356,14 +354,8 @@ def check_and_set_ln_bits_keys(identifier, npub): os.getenv("LNADDRESS_" + identifier.upper())) - - - - def add_key_to_env_file(value, oskey): env_path = Path('.env') if env_path.is_file(): dotenv.load_dotenv(env_path, verbose=True, override=True) - dotenv.set_key(env_path, value, oskey) - - + dotenv.set_key(env_path, value, oskey) \ No newline at end of file diff --git a/setup.py b/setup.py index 2ad52c7..abdd287 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.7' +VERSION = '0.0.8' DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') From 0062d785a633258ad8eba12f71e8bd18c4b12cfd Mon Sep 17 00:00:00 2001 From: Believethehype Date: Tue, 19 Dec 2023 15:46:30 +0100 Subject: [PATCH 14/16] adding examples, minor fixes --- examples/ollama_dvm/README.md | 26 ++++++ examples/ollama_dvm/main.py | 54 +++++++++++++ examples/ollama_dvm/test_client.py | 94 ++++++++++++++++++++++ examples/tts_dvm/README.md | 26 ++++++ examples/tts_dvm/main.py | 60 ++++++++++++++ examples/tts_dvm/test_client.py | 98 +++++++++++++++++++++++ main.py | 2 +- nostr_dvm/dvm.py | 8 +- nostr_dvm/tasks/textgeneration_llmlite.py | 16 ++-- nostr_dvm/utils/definitions.py | 2 +- setup.py | 2 +- 11 files changed, 372 insertions(+), 16 deletions(-) create mode 100644 examples/ollama_dvm/README.md create mode 100644 examples/ollama_dvm/main.py create mode 100644 examples/ollama_dvm/test_client.py create mode 100644 examples/tts_dvm/README.md create mode 100644 examples/tts_dvm/main.py create mode 100644 examples/tts_dvm/test_client.py diff --git a/examples/ollama_dvm/README.md b/examples/ollama_dvm/README.md new file mode 100644 index 0000000..2709125 --- /dev/null +++ b/examples/ollama_dvm/README.md @@ -0,0 +1,26 @@ +# NostrAI: Nostr NIP90 Data Vending Machine Framework Example + +Projects in this folder contain ready-to-use DVMs. To tun the DVM following the next steps: + +## To get started: +- Install Python 3.10 + + +Create a new venv in this directory by opening the terminal here, or navigate to this directory and type: `"python -m venv venv"` + - Place .env file (based on .env_example) in this folder. + - Recommended but optional: + - Create a `LNbits` account on an accessible instance of your choice, enter one account's id and admin key (this account will create other accounts for the dvms) Open the .env file and enter this info to `LNBITS_ADMIN_KEY`, `LNBITS_ADMIN_ID`, `LNBITS_HOST`. + - If you are running an own instance of `Nostdress` enter `NOSTDRESS_DOMAIN` or use the default one. + - Activate the venv with + - MacOS/Linux: source ./venv/bin/activate + - Windows: .\venv\Scripts\activate + - Type: `pip install nostr-dvm` + - Run `python3 main.py` (or python main.py) + - The framework will then automatically create keys, nip89 tags and zapable NIP57 `lightning addresses` for your dvms in this file. + - Check the .env file if these values look correct. + - Check the `main.py` file. You can update the image/description/name of your DVM before announcing it. + - You can then in main.py set `admin_config.REBROADCAST_NIP89` and + `admin_config.UPDATE_PROFILE` to `True` to announce the NIP89 info and update the npubs profile automatically. + - After this was successful you can set these back to False until the next time you want to update the NIP89 or profile. + +You are now running your own DVM. \ No newline at end of file diff --git a/examples/ollama_dvm/main.py b/examples/ollama_dvm/main.py new file mode 100644 index 0000000..6e0fb91 --- /dev/null +++ b/examples/ollama_dvm/main.py @@ -0,0 +1,54 @@ +import json +from pathlib import Path +import dotenv + +from nostr_dvm.tasks.textgeneration_llmlite import TextGenerationLLMLite +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.dvmconfig import build_default_config +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag + + +def main(): + identifier = "llama2" + name = "Ollama" + + dvm_config = build_default_config(identifier) + admin_config = AdminConfig() + admin_config.REBROADCAST_NIP89 = False + admin_config.UPDATE_PROFILE = False + admin_config.LUD16 = dvm_config.LN_ADDRESS + + options = {'default_model': "ollama/llama2", 'server': "http://localhost:11434"} + + nip89info = { + "name": name, + "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", + "about": "I use a LLM connected via OLLAMA", + "encryptionSupported": True, + "cashuAccepted": True, + "nip90Params": { + + } + } + + nip89config = NIP89Config() + nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) + nip89config.CONTENT = json.dumps(nip89info) + + ollama = TextGenerationLLMLite(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, + options=options) + ollama.run() + + +if __name__ == '__main__': + env_path = Path('.env') + if not env_path.is_file(): + with open('.env', 'w') as f: + print("Writing new .env file") + f.write('') + if env_path.is_file(): + print(f'loading environment from {env_path.resolve()}') + dotenv.load_dotenv(env_path, verbose=True, override=True) + else: + raise FileNotFoundError(f'.env file not found at {env_path} ') + main() diff --git a/examples/ollama_dvm/test_client.py b/examples/ollama_dvm/test_client.py new file mode 100644 index 0000000..05c3752 --- /dev/null +++ b/examples/ollama_dvm/test_client.py @@ -0,0 +1,94 @@ +import json +import time +from pathlib import Path +from threading import Thread + +import dotenv +from nostr_sdk import Keys, Client, Tag, EventBuilder, Filter, HandleNotification, Timestamp, nip04_decrypt + +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nostr_utils import send_event, check_and_set_private_key +from nostr_dvm.utils.definitions import EventDefinitions + + +def nostr_client_test_llm(prompt): + keys = Keys.from_sk_str(check_and_set_private_key("test_client")) + + iTag = Tag.parse(["i", prompt, "text"]) + relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"]) + alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to generate TTSt"]) + event = EventBuilder(EventDefinitions.KIND_NIP90_GENERATE_TEXT, str("Generate an Audio File."), + [iTag, relaysTag, alttag]).to_event(keys) + + relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"] + + client = Client(keys) + for relay in relay_list: + client.add_relay(relay) + client.connect() + config = DVMConfig + send_event(event, client=client, dvm_config=config) + return event.as_json() + +def nostr_client(): + keys = Keys.from_sk_str(check_and_set_private_key("test_client")) + sk = keys.secret_key() + pk = keys.public_key() + print(f"Nostr Test Client public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ") + client = Client(keys) + dvmconfig = DVMConfig() + for relay in dvmconfig.RELAY_LIST: + client.add_relay(relay) + client.connect() + + dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM, + EventDefinitions.KIND_ZAP]).since( + Timestamp.now()) # events to us specific + dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_RESULT_GENERATE_TEXT, + EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events + client.subscribe([dm_zap_filter, dvm_filter]) + + + nostr_client_test_llm("Tell me a joke about a purple Ostrich!") + print("Sending Job Request") + + + #nostr_client_test_image_private("a beautiful ostrich watching the sunset") + class NotificationHandler(HandleNotification): + def handle(self, relay_url, event): + print(f"Received new event from {relay_url}: {event.as_json()}") + if event.kind() == 7000: + print("[Nostr Client]: " + event.as_json()) + elif 6000 < event.kind() < 6999: + print("[Nostr Client]: " + event.as_json()) + print("[Nostr Client]: " + event.content()) + + elif event.kind() == 4: + dec_text = nip04_decrypt(sk, event.pubkey(), event.content()) + print("[Nostr Client]: " + f"Received new msg: {dec_text}") + + elif event.kind() == 9735: + print("[Nostr Client]: " + f"Received new zap:") + print(event.as_json()) + + def handle_msg(self, relay_url, msg): + return + + client.handle_notifications(NotificationHandler()) + while True: + time.sleep(5.0) + + +if __name__ == '__main__': + + env_path = Path('.env') + if env_path.is_file(): + print(f'loading environment from {env_path.resolve()}') + dotenv.load_dotenv(env_path, verbose=True, override=True) + else: + raise FileNotFoundError(f'.env file not found at {env_path} ') + + nostr_dvm_thread = Thread(target=nostr_client()) + nostr_dvm_thread.start() diff --git a/examples/tts_dvm/README.md b/examples/tts_dvm/README.md new file mode 100644 index 0000000..2709125 --- /dev/null +++ b/examples/tts_dvm/README.md @@ -0,0 +1,26 @@ +# NostrAI: Nostr NIP90 Data Vending Machine Framework Example + +Projects in this folder contain ready-to-use DVMs. To tun the DVM following the next steps: + +## To get started: +- Install Python 3.10 + + +Create a new venv in this directory by opening the terminal here, or navigate to this directory and type: `"python -m venv venv"` + - Place .env file (based on .env_example) in this folder. + - Recommended but optional: + - Create a `LNbits` account on an accessible instance of your choice, enter one account's id and admin key (this account will create other accounts for the dvms) Open the .env file and enter this info to `LNBITS_ADMIN_KEY`, `LNBITS_ADMIN_ID`, `LNBITS_HOST`. + - If you are running an own instance of `Nostdress` enter `NOSTDRESS_DOMAIN` or use the default one. + - Activate the venv with + - MacOS/Linux: source ./venv/bin/activate + - Windows: .\venv\Scripts\activate + - Type: `pip install nostr-dvm` + - Run `python3 main.py` (or python main.py) + - The framework will then automatically create keys, nip89 tags and zapable NIP57 `lightning addresses` for your dvms in this file. + - Check the .env file if these values look correct. + - Check the `main.py` file. You can update the image/description/name of your DVM before announcing it. + - You can then in main.py set `admin_config.REBROADCAST_NIP89` and + `admin_config.UPDATE_PROFILE` to `True` to announce the NIP89 info and update the npubs profile automatically. + - After this was successful you can set these back to False until the next time you want to update the NIP89 or profile. + +You are now running your own DVM. \ No newline at end of file diff --git a/examples/tts_dvm/main.py b/examples/tts_dvm/main.py new file mode 100644 index 0000000..2ce8d46 --- /dev/null +++ b/examples/tts_dvm/main.py @@ -0,0 +1,60 @@ +import json +from pathlib import Path +import dotenv +from nostr_dvm.tasks.texttospeech import TextToSpeech +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.dvmconfig import build_default_config +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag + + +def main(): + identifier = "tts" + name = "Guy Swann Clone" + + dvm_config = build_default_config(identifier) + admin_config = AdminConfig() + admin_config.REBROADCAST_NIP89 = False + admin_config.UPDATE_PROFILE = False + admin_config.LUD16 = dvm_config.LN_ADDRESS + + # Use default file if paramter is empty, else overwrite with any local wav file + options = {'input_file': ""} + + nip89info = { + "name": name, + "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", + "about": "I Generate Speech from Text", + "encryptionSupported": True, + "cashuAccepted": True, + "nip90Params": { + "language": { + "required": False, + "values": [] + } + } + } + + nip89config = NIP89Config() + nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) + nip89config.CONTENT = json.dumps(nip89info) + + tts = TextToSpeech(name=name, + dvm_config=dvm_config, + nip89config=nip89config, + admin_config=admin_config, + options=options) + tts.run() + + +if __name__ == '__main__': + env_path = Path('.env') + if not env_path.is_file(): + with open('.env', 'w') as f: + print("Writing new .env file") + f.write('') + if env_path.is_file(): + print(f'loading environment from {env_path.resolve()}') + dotenv.load_dotenv(env_path, verbose=True, override=True) + else: + raise FileNotFoundError(f'.env file not found at {env_path} ') + main() diff --git a/examples/tts_dvm/test_client.py b/examples/tts_dvm/test_client.py new file mode 100644 index 0000000..47cc76e --- /dev/null +++ b/examples/tts_dvm/test_client.py @@ -0,0 +1,98 @@ +import json +import time +from pathlib import Path +from threading import Thread + +import dotenv +from nostr_sdk import Keys, Client, Tag, EventBuilder, Filter, HandleNotification, Timestamp, nip04_decrypt + +from nostr_dvm.utils.dvmconfig import DVMConfig +from nostr_dvm.utils.nostr_utils import send_event, check_and_set_private_key +from nostr_dvm.utils.definitions import EventDefinitions + + +def nostr_client_test_tts(prompt): + keys = Keys.from_sk_str(check_and_set_private_key("test_client")) + + iTag = Tag.parse(["i", prompt, "text"]) + paramTag1 = Tag.parse(["param", "language", "en"]) + + + bidTag = Tag.parse(['bid', str(1000 * 1000), str(1000 * 1000)]) + relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"]) + alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to generate TTSt"]) + event = EventBuilder(EventDefinitions.KIND_NIP90_TEXT_TO_SPEECH, str("Generate an Audio File."), + [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys) + + relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"] + + client = Client(keys) + for relay in relay_list: + client.add_relay(relay) + client.connect() + config = DVMConfig + send_event(event, client=client, dvm_config=config) + return event.as_json() + +def nostr_client(): + keys = Keys.from_sk_str(check_and_set_private_key("test_client")) + sk = keys.secret_key() + pk = keys.public_key() + print(f"Nostr Test Client public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ") + client = Client(keys) + dvmconfig = DVMConfig() + for relay in dvmconfig.RELAY_LIST: + client.add_relay(relay) + client.connect() + + dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM, + EventDefinitions.KIND_ZAP]).since( + Timestamp.now()) # events to us specific + dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_RESULT_TEXT_TO_SPEECH, + EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events + client.subscribe([dm_zap_filter, dvm_filter]) + + + nostr_client_test_tts("Hello, this is a test. Mic check one, two.") + print("Sending Job Request") + + + #nostr_client_test_image_private("a beautiful ostrich watching the sunset") + class NotificationHandler(HandleNotification): + def handle(self, relay_url, event): + print(f"Received new event from {relay_url}: {event.as_json()}") + if event.kind() == 7000: + print("[Nostr Client]: " + event.as_json()) + elif 6000 < event.kind() < 6999: + print("[Nostr Client]: " + event.as_json()) + print("[Nostr Client]: " + event.content()) + + elif event.kind() == 4: + dec_text = nip04_decrypt(sk, event.pubkey(), event.content()) + print("[Nostr Client]: " + f"Received new msg: {dec_text}") + + elif event.kind() == 9735: + print("[Nostr Client]: " + f"Received new zap:") + print(event.as_json()) + + def handle_msg(self, relay_url, msg): + return + + client.handle_notifications(NotificationHandler()) + while True: + time.sleep(5.0) + + +if __name__ == '__main__': + + env_path = Path('.env') + if env_path.is_file(): + print(f'loading environment from {env_path.resolve()}') + dotenv.load_dotenv(env_path, verbose=True, override=True) + else: + raise FileNotFoundError(f'.env file not found at {env_path} ') + + nostr_dvm_thread = Thread(target=nostr_client()) + nostr_dvm_thread.start() diff --git a/main.py b/main.py index 6a2bf38..2d71129 100644 --- a/main.py +++ b/main.py @@ -148,7 +148,7 @@ if __name__ == '__main__': env_path = Path('.env') if not env_path.is_file(): with open('.env', 'w') as f: - print("Wrting new .env file") + print("Writing new .env file") f.write('') if env_path.is_file(): print(f'loading environment from {env_path.resolve()}') diff --git a/nostr_dvm/dvm.py b/nostr_dvm/dvm.py index 32f899f..44febf7 100644 --- a/nostr_dvm/dvm.py +++ b/nostr_dvm/dvm.py @@ -484,9 +484,11 @@ class DVM: print("Finished processing, loading data..") with open(os.path.abspath('output.txt')) as f: - result = f.readlines()[0] - print(result) - #f.close() + resultall = f.readlines() + result = "" + for line in resultall: + if line != '\n': + result += line os.remove(os.path.abspath('output.txt')) else: #Some components might have issues with running code in otuside venv. # We install locally in these cases for now diff --git a/nostr_dvm/tasks/textgeneration_llmlite.py b/nostr_dvm/tasks/textgeneration_llmlite.py index 9bee4c1..ab2bd0e 100644 --- a/nostr_dvm/tasks/textgeneration_llmlite.py +++ b/nostr_dvm/tasks/textgeneration_llmlite.py @@ -20,7 +20,7 @@ Outputs: Generated text """ -class TextGenerationOLLAMA(DVMTaskInterface): +class TextGenerationLLMLite(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_GENERATE_TEXT TASK: str = "text-to-text" FIX_COST: float = 0 @@ -80,7 +80,8 @@ class TextGenerationOLLAMA(DVMTaskInterface): response = completion( model=options["model"], messages=[{"content": options["prompt"], "role": "user"}], - api_base=options["server"] + api_base=options["server"], + stream=False ) print(response.choices[0].message.content) return response.choices[0].message.content @@ -112,25 +113,20 @@ def build_example(name, identifier, admin_config): "about": "I use a LLM connected via OLLAMA", "encryptionSupported": True, "cashuAccepted": True, - "nip90Params": { - "size": { - "required": False, - "values": ["1024:1024", "1024x1792", "1792x1024"] - } - } + "nip90Params": {} } nip89config = NIP89Config() nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) nip89config.CONTENT = json.dumps(nip89info) - return TextGenerationOLLAMA(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, options=options) + return TextGenerationLLMLite(name=name, dvm_config=dvm_config, nip89config=nip89config, admin_config=admin_config, options=options) def process_venv(): args = DVMTaskInterface.process_args() dvm_config = build_default_config(args.identifier) - dvm = TextGenerationOLLAMA(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + dvm = TextGenerationLLMLite(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) result = dvm.process(json.loads(args.request)) DVMTaskInterface.write_output(result, args.output) diff --git a/nostr_dvm/utils/definitions.py b/nostr_dvm/utils/definitions.py index 5191ff0..99d1df9 100644 --- a/nostr_dvm/utils/definitions.py +++ b/nostr_dvm/utils/definitions.py @@ -15,7 +15,7 @@ class EventDefinitions: KIND_NIP90_TRANSLATE_TEXT = 5002 KIND_NIP90_RESULT_TRANSLATE_TEXT = KIND_NIP90_TRANSLATE_TEXT + 1000 KIND_NIP90_TEXT_TO_SPEECH = 5005 - KIND_NIP90_TEXT_TO_SPEECH_RESULT = KIND_NIP90_TEXT_TO_SPEECH + 1000 + KIND_NIP90_RESULT_TEXT_TO_SPEECH = KIND_NIP90_TEXT_TO_SPEECH + 1000 KIND_NIP90_GENERATE_TEXT = 5050 KIND_NIP90_RESULT_GENERATE_TEXT = KIND_NIP90_GENERATE_TEXT + 1000 KIND_NIP90_GENERATE_IMAGE = 5100 diff --git a/setup.py b/setup.py index abdd287..628e364 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.8' +VERSION = '0.0.9' DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') From 64710b4d1d4c42f9ed7b9c91c6226e02263ed3f9 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Wed, 20 Dec 2023 09:32:23 +0100 Subject: [PATCH 15/16] added .env_example for examples --- examples/ollama_dvm/.env_example | 17 +++++++++++++++++ examples/tts_dvm/.env_example | 15 +++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 examples/ollama_dvm/.env_example create mode 100644 examples/tts_dvm/.env_example diff --git a/examples/ollama_dvm/.env_example b/examples/ollama_dvm/.env_example new file mode 100644 index 0000000..eff18ab --- /dev/null +++ b/examples/ollama_dvm/.env_example @@ -0,0 +1,17 @@ +#Create an account with a lnbits instance of your choice, add the admin key and id here. This account will be used to create a new lnbits wallet for each dvm/bot +LNBITS_ADMIN_KEY = "" +LNBITS_ADMIN_ID = "" +LNBITS_HOST = "https://lnbits.com" #Use your own/a trusted instance ideally. +# In order to create a zappable lightning address, host nostdress on your domain or use this preinstalled domain. +# We will use the api to create and manage zapable lightning addresses +NOSTDRESS_DOMAIN = "nostrdvm.com" + +#Backend Specific Options for tasks that require them. A DVM needing these should only be started if these are set. +OPENAI_API_KEY = "" # Enter your OpenAI API Key to use DVMs with OpenAI services + +# We will automatically create dtags and private keys based on the identifier variable in main. +# If your DVM already has a dtag and private key you can replace it here before publishing the DTAG to not create a new one. +# The name and NIP90 info of the DVM can be changed but the identifier must stay the same in order to not create a different dtag. + +# We will also create new wallets on the given lnbits instance for each dvm. If you want to use an existing wallet, you can replace the parameters here as well. +# Make sure you backup this file to keep access to your wallets \ No newline at end of file diff --git a/examples/tts_dvm/.env_example b/examples/tts_dvm/.env_example new file mode 100644 index 0000000..ef4a5e4 --- /dev/null +++ b/examples/tts_dvm/.env_example @@ -0,0 +1,15 @@ +#Create an account with a lnbits instance of your choice, add the admin key and id here. This account will be used to create a new lnbits wallet for each dvm/bot +LNBITS_ADMIN_KEY = "" +LNBITS_ADMIN_ID = "" +LNBITS_HOST = "https://lnbits.com" #Use your own/a trusted instance ideally. +# In order to create a zappable lightning address, host nostdress on your domain or use this preinstalled domain. +# We will use the api to create and manage zapable lightning addresses +NOSTDRESS_DOMAIN = "nostrdvm.com" + + +# We will automatically create dtags and private keys based on the identifier variable in main. +# If your DVM already has a dtag and private key you can replace it here before publishing the DTAG to not create a new one. +# The name and NIP90 info of the DVM can be changed but the identifier must stay the same in order to not create a different dtag. + +# We will also create new wallets on the given lnbits instance for each dvm. If you want to use an existing wallet, you can replace the parameters here as well. +# Make sure you backup this file to keep access to your wallets \ No newline at end of file From 15d3384dceb2a97fbb782778b1cf4f8d1b52bb87 Mon Sep 17 00:00:00 2001 From: Believethehype Date: Wed, 20 Dec 2023 11:59:42 +0100 Subject: [PATCH 16/16] adding MLX backend and example for stable diffusion 2.1 from https://github.com/ml-explore/mlx-examples --- backends/__init__.py | 0 backends/mlx/__init__.py | 0 backends/mlx/stable_diffusion/__init__.py | 0 backends/mlx/stable_diffusion/clip.py | 70 ++++ backends/mlx/stable_diffusion/config.py | 48 +++ backends/mlx/stable_diffusion/model_io.py | 292 ++++++++++++++ backends/mlx/stable_diffusion/sampler.py | 74 ++++ backends/mlx/stable_diffusion/tokenizer.py | 100 +++++ backends/mlx/stable_diffusion/unet.py | 425 +++++++++++++++++++++ backends/mlx/stable_diffusion/vae.py | 268 +++++++++++++ main.py | 11 +- nostr_dvm/tasks/imagegeneration_mlx.py | 186 +++++++++ nostr_dvm/utils/zap_utils.py | 4 - setup.py | 10 +- tests/test_dvm_client.py | 4 +- 15 files changed, 1481 insertions(+), 11 deletions(-) create mode 100644 backends/__init__.py create mode 100644 backends/mlx/__init__.py create mode 100644 backends/mlx/stable_diffusion/__init__.py create mode 100644 backends/mlx/stable_diffusion/clip.py create mode 100644 backends/mlx/stable_diffusion/config.py create mode 100644 backends/mlx/stable_diffusion/model_io.py create mode 100644 backends/mlx/stable_diffusion/sampler.py create mode 100644 backends/mlx/stable_diffusion/tokenizer.py create mode 100644 backends/mlx/stable_diffusion/unet.py create mode 100644 backends/mlx/stable_diffusion/vae.py create mode 100644 nostr_dvm/tasks/imagegeneration_mlx.py diff --git a/backends/__init__.py b/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backends/mlx/__init__.py b/backends/mlx/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backends/mlx/stable_diffusion/__init__.py b/backends/mlx/stable_diffusion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backends/mlx/stable_diffusion/clip.py b/backends/mlx/stable_diffusion/clip.py new file mode 100644 index 0000000..49d7b51 --- /dev/null +++ b/backends/mlx/stable_diffusion/clip.py @@ -0,0 +1,70 @@ +# Copyright © 2023 Apple Inc. + +import mlx.core as mx +import mlx.nn as nn + +from .config import CLIPTextModelConfig + + +class CLIPEncoderLayer(nn.Module): + """The transformer encoder layer from CLIP.""" + + def __init__(self, model_dims: int, num_heads: int): + super().__init__() + + self.layer_norm1 = nn.LayerNorm(model_dims) + self.layer_norm2 = nn.LayerNorm(model_dims) + + self.attention = nn.MultiHeadAttention(model_dims, num_heads) + # Add biases to the attention projections to match CLIP + self.attention.query_proj.bias = mx.zeros(model_dims) + self.attention.key_proj.bias = mx.zeros(model_dims) + self.attention.value_proj.bias = mx.zeros(model_dims) + self.attention.out_proj.bias = mx.zeros(model_dims) + + self.linear1 = nn.Linear(model_dims, 4 * model_dims) + self.linear2 = nn.Linear(4 * model_dims, model_dims) + + def __call__(self, x, attn_mask=None): + y = self.layer_norm1(x) + y = self.attention(y, y, y, attn_mask) + x = y + x + + y = self.layer_norm2(x) + y = self.linear1(y) + y = nn.gelu_approx(y) + y = self.linear2(y) + x = y + x + + return x + + +class CLIPTextModel(nn.Module): + """Implements the text encoder transformer from CLIP.""" + + def __init__(self, config: CLIPTextModelConfig): + super().__init__() + + self.token_embedding = nn.Embedding(config.vocab_size, config.model_dims) + self.position_embedding = nn.Embedding(config.max_length, config.model_dims) + self.layers = [ + CLIPEncoderLayer(config.model_dims, config.num_heads) + for i in range(config.num_layers) + ] + self.final_layer_norm = nn.LayerNorm(config.model_dims) + + def __call__(self, x): + # Extract some shapes + B, N = x.shape + + # Compute the embeddings + x = self.token_embedding(x) + x = x + self.position_embedding.weight[:N] + + # Compute the features from the transformer + mask = nn.MultiHeadAttention.create_additive_causal_mask(N, x.dtype) + for l in self.layers: + x = l(x, mask) + + # Apply the final layernorm and return + return self.final_layer_norm(x) diff --git a/backends/mlx/stable_diffusion/config.py b/backends/mlx/stable_diffusion/config.py new file mode 100644 index 0000000..6fcf595 --- /dev/null +++ b/backends/mlx/stable_diffusion/config.py @@ -0,0 +1,48 @@ +# Copyright © 2023 Apple Inc. + +from dataclasses import dataclass +from typing import Optional, Tuple + + +@dataclass +class AutoencoderConfig: + in_channels: int = 3 + out_channels: int = 3 + latent_channels_out: int = 8 + latent_channels_in: int = 4 + block_out_channels: Tuple[int] = (128, 256, 512, 512) + layers_per_block: int = 2 + norm_num_groups: int = 32 + scaling_factor: float = 0.18215 + + +@dataclass +class CLIPTextModelConfig: + num_layers: int = 23 + model_dims: int = 1024 + num_heads: int = 16 + max_length: int = 77 + vocab_size: int = 49408 + + +@dataclass +class UNetConfig: + in_channels: int = 4 + out_channels: int = 4 + conv_in_kernel: int = 3 + conv_out_kernel: int = 3 + block_out_channels: Tuple[int] = (320, 640, 1280, 1280) + layers_per_block: Tuple[int] = (2, 2, 2, 2) + mid_block_layers: int = 2 + transformer_layers_per_block: Tuple[int] = (1, 1, 1, 1) + num_attention_heads: Tuple[int] = (5, 10, 20, 20) + cross_attention_dim: Tuple[int] = (1024,) * 4 + norm_num_groups: int = 32 + + +@dataclass +class DiffusionConfig: + beta_schedule: str = "scaled_linear" + beta_start: float = 0.00085 + beta_end: float = 0.012 + num_train_steps: int = 1000 diff --git a/backends/mlx/stable_diffusion/model_io.py b/backends/mlx/stable_diffusion/model_io.py new file mode 100644 index 0000000..57879ef --- /dev/null +++ b/backends/mlx/stable_diffusion/model_io.py @@ -0,0 +1,292 @@ +# Copyright © 2023 Apple Inc. + +import json +from functools import partial + +import numpy as np +from huggingface_hub import hf_hub_download +from safetensors import safe_open as safetensor_open + +import mlx.core as mx +from mlx.utils import tree_unflatten + +from .clip import CLIPTextModel +from .config import UNetConfig, CLIPTextModelConfig, AutoencoderConfig, DiffusionConfig +from .tokenizer import Tokenizer +from .unet import UNetModel +from .vae import Autoencoder + + +_DEFAULT_MODEL = "stabilityai/stable-diffusion-2-1-base" +_MODELS = { + # See https://huggingface.co/stabilityai/stable-diffusion-2-1-base for the model details and license + "stabilityai/stable-diffusion-2-1-base": { + "unet_config": "unet/config.json", + "unet": "unet/diffusion_pytorch_model.safetensors", + "text_encoder_config": "text_encoder/config.json", + "text_encoder": "text_encoder/model.safetensors", + "vae_config": "vae/config.json", + "vae": "vae/diffusion_pytorch_model.safetensors", + "diffusion_config": "scheduler/scheduler_config.json", + "tokenizer_vocab": "tokenizer/vocab.json", + "tokenizer_merges": "tokenizer/merges.txt", + } +} + + +def _from_numpy(x): + return mx.array(np.ascontiguousarray(x)) + + +def map_unet_weights(key, value): + # Map up/downsampling + if "downsamplers" in key: + key = key.replace("downsamplers.0.conv", "downsample") + if "upsamplers" in key: + key = key.replace("upsamplers.0.conv", "upsample") + + # Map the mid block + if "mid_block.resnets.0" in key: + key = key.replace("mid_block.resnets.0", "mid_blocks.0") + if "mid_block.attentions.0" in key: + key = key.replace("mid_block.attentions.0", "mid_blocks.1") + if "mid_block.resnets.1" in key: + key = key.replace("mid_block.resnets.1", "mid_blocks.2") + + # Map attention layers + if "to_k" in key: + key = key.replace("to_k", "key_proj") + if "to_out.0" in key: + key = key.replace("to_out.0", "out_proj") + if "to_q" in key: + key = key.replace("to_q", "query_proj") + if "to_v" in key: + key = key.replace("to_v", "value_proj") + + # Map transformer ffn + if "ff.net.2" in key: + key = key.replace("ff.net.2", "linear3") + if "ff.net.0" in key: + k1 = key.replace("ff.net.0.proj", "linear1") + k2 = key.replace("ff.net.0.proj", "linear2") + v1, v2 = np.split(value, 2) + + return [(k1, _from_numpy(v1)), (k2, _from_numpy(v2))] + + if "conv_shortcut.weight" in key: + value = value.squeeze() + + # Transform the weights from 1x1 convs to linear + if len(value.shape) == 4 and ("proj_in" in key or "proj_out" in key): + value = value.squeeze() + + if len(value.shape) == 4: + value = value.transpose(0, 2, 3, 1) + + return [(key, _from_numpy(value))] + + +def map_clip_text_encoder_weights(key, value): + # Remove prefixes + if key.startswith("text_model."): + key = key[11:] + if key.startswith("embeddings."): + key = key[11:] + if key.startswith("encoder."): + key = key[8:] + + # Map attention layers + if "self_attn." in key: + key = key.replace("self_attn.", "attention.") + if "q_proj." in key: + key = key.replace("q_proj.", "query_proj.") + if "k_proj." in key: + key = key.replace("k_proj.", "key_proj.") + if "v_proj." in key: + key = key.replace("v_proj.", "value_proj.") + + # Map ffn layers + if "mlp.fc1" in key: + key = key.replace("mlp.fc1", "linear1") + if "mlp.fc2" in key: + key = key.replace("mlp.fc2", "linear2") + + return [(key, _from_numpy(value))] + + +def map_vae_weights(key, value): + # Map up/downsampling + if "downsamplers" in key: + key = key.replace("downsamplers.0.conv", "downsample") + if "upsamplers" in key: + key = key.replace("upsamplers.0.conv", "upsample") + + # Map attention layers + if "to_k" in key: + key = key.replace("to_k", "key_proj") + if "to_out.0" in key: + key = key.replace("to_out.0", "out_proj") + if "to_q" in key: + key = key.replace("to_q", "query_proj") + if "to_v" in key: + key = key.replace("to_v", "value_proj") + + # Map the mid block + if "mid_block.resnets.0" in key: + key = key.replace("mid_block.resnets.0", "mid_blocks.0") + if "mid_block.attentions.0" in key: + key = key.replace("mid_block.attentions.0", "mid_blocks.1") + if "mid_block.resnets.1" in key: + key = key.replace("mid_block.resnets.1", "mid_blocks.2") + + # Map the quant/post_quant layers + if "quant_conv" in key: + key = key.replace("quant_conv", "quant_proj") + value = value.squeeze() + + # Map the conv_shortcut to linear + if "conv_shortcut.weight" in key: + value = value.squeeze() + + if len(value.shape) == 4: + value = value.transpose(0, 2, 3, 1) + + return [(key, _from_numpy(value))] + + +def _flatten(params): + return [(k, v) for p in params for (k, v) in p] + + +def _load_safetensor_weights(mapper, model, weight_file, float16: bool = False): + dtype = np.float16 if float16 else np.float32 + with safetensor_open(weight_file, framework="numpy") as f: + weights = _flatten([mapper(k, f.get_tensor(k).astype(dtype)) for k in f.keys()]) + model.update(tree_unflatten(weights)) + + +def _check_key(key: str, part: str): + if key not in _MODELS: + raise ValueError( + f"[{part}] '{key}' model not found, choose one of {{{','.join(_MODELS.keys())}}}" + ) + + +def load_unet(key: str = _DEFAULT_MODEL, float16: bool = False): + """Load the stable diffusion UNet from Hugging Face Hub.""" + _check_key(key, "load_unet") + + # Download the config and create the model + unet_config = _MODELS[key]["unet_config"] + with open(hf_hub_download(key, unet_config)) as f: + config = json.load(f) + + n_blocks = len(config["block_out_channels"]) + model = UNetModel( + UNetConfig( + in_channels=config["in_channels"], + out_channels=config["out_channels"], + block_out_channels=config["block_out_channels"], + layers_per_block=[config["layers_per_block"]] * n_blocks, + num_attention_heads=[config["attention_head_dim"]] * n_blocks + if isinstance(config["attention_head_dim"], int) + else config["attention_head_dim"], + cross_attention_dim=[config["cross_attention_dim"]] * n_blocks, + norm_num_groups=config["norm_num_groups"], + ) + ) + + # Download the weights and map them into the model + unet_weights = _MODELS[key]["unet"] + weight_file = hf_hub_download(key, unet_weights) + _load_safetensor_weights(map_unet_weights, model, weight_file, float16) + + return model + + +def load_text_encoder(key: str = _DEFAULT_MODEL, float16: bool = False): + """Load the stable diffusion text encoder from Hugging Face Hub.""" + _check_key(key, "load_text_encoder") + + # Download the config and create the model + text_encoder_config = _MODELS[key]["text_encoder_config"] + with open(hf_hub_download(key, text_encoder_config)) as f: + config = json.load(f) + + model = CLIPTextModel( + CLIPTextModelConfig( + num_layers=config["num_hidden_layers"], + model_dims=config["hidden_size"], + num_heads=config["num_attention_heads"], + max_length=config["max_position_embeddings"], + vocab_size=config["vocab_size"], + ) + ) + + # Download the weights and map them into the model + text_encoder_weights = _MODELS[key]["text_encoder"] + weight_file = hf_hub_download(key, text_encoder_weights) + _load_safetensor_weights(map_clip_text_encoder_weights, model, weight_file, float16) + + return model + + +def load_autoencoder(key: str = _DEFAULT_MODEL, float16: bool = False): + """Load the stable diffusion autoencoder from Hugging Face Hub.""" + _check_key(key, "load_autoencoder") + + # Download the config and create the model + vae_config = _MODELS[key]["vae_config"] + with open(hf_hub_download(key, vae_config)) as f: + config = json.load(f) + + model = Autoencoder( + AutoencoderConfig( + in_channels=config["in_channels"], + out_channels=config["out_channels"], + latent_channels_out=2 * config["latent_channels"], + latent_channels_in=config["latent_channels"], + block_out_channels=config["block_out_channels"], + layers_per_block=config["layers_per_block"], + norm_num_groups=config["norm_num_groups"], + ) + ) + + # Download the weights and map them into the model + vae_weights = _MODELS[key]["vae"] + weight_file = hf_hub_download(key, vae_weights) + _load_safetensor_weights(map_vae_weights, model, weight_file, float16) + + return model + + +def load_diffusion_config(key: str = _DEFAULT_MODEL): + """Load the stable diffusion config from Hugging Face Hub.""" + _check_key(key, "load_diffusion_config") + + diffusion_config = _MODELS[key]["diffusion_config"] + with open(hf_hub_download(key, diffusion_config)) as f: + config = json.load(f) + + return DiffusionConfig( + beta_start=config["beta_start"], + beta_end=config["beta_end"], + beta_schedule=config["beta_schedule"], + num_train_steps=config["num_train_timesteps"], + ) + + +def load_tokenizer(key: str = _DEFAULT_MODEL): + _check_key(key, "load_tokenizer") + + vocab_file = hf_hub_download(key, _MODELS[key]["tokenizer_vocab"]) + with open(vocab_file, encoding="utf-8") as f: + vocab = json.load(f) + + merges_file = hf_hub_download(key, _MODELS[key]["tokenizer_merges"]) + with open(merges_file, encoding="utf-8") as f: + bpe_merges = f.read().strip().split("\n")[1 : 49152 - 256 - 2 + 1] + bpe_merges = [tuple(m.split()) for m in bpe_merges] + bpe_ranks = dict(map(reversed, enumerate(bpe_merges))) + + return Tokenizer(bpe_ranks, vocab) diff --git a/backends/mlx/stable_diffusion/sampler.py b/backends/mlx/stable_diffusion/sampler.py new file mode 100644 index 0000000..a1edf93 --- /dev/null +++ b/backends/mlx/stable_diffusion/sampler.py @@ -0,0 +1,74 @@ +# Copyright © 2023 Apple Inc. + +from .config import DiffusionConfig + +import mlx.core as mx + + +def _linspace(a, b, num): + x = mx.arange(0, num) / (num - 1) + return (b - a) * x + a + + +def _interp(y, x_new): + """Interpolate the function defined by (arange(0, len(y)), y) at positions x_new.""" + x_low = x_new.astype(mx.int32) + x_high = mx.minimum(x_low + 1, len(y) - 1) + + y_low = y[x_low] + y_high = y[x_high] + delta_x = x_new - x_low + y_new = y_low * (1 - delta_x) + delta_x * y_high + + return y_new + + +class SimpleEulerSampler: + """A simple Euler integrator that can be used to sample from our diffusion models. + + The method ``step()`` performs one Euler step from x_t to x_t_prev. + """ + + def __init__(self, config: DiffusionConfig): + # Compute the noise schedule + if config.beta_schedule == "linear": + betas = _linspace( + config.beta_start, config.beta_end, config.num_train_steps + ) + elif config.beta_schedule == "scaled_linear": + betas = _linspace( + config.beta_start**0.5, config.beta_end**0.5, config.num_train_steps + ).square() + else: + raise NotImplementedError(f"{config.beta_schedule} is not implemented.") + + alphas = 1 - betas + alphas_cumprod = mx.cumprod(alphas) + + self._sigmas = mx.concatenate( + [mx.zeros(1), ((1 - alphas_cumprod) / alphas_cumprod).sqrt()] + ) + + def sample_prior(self, shape, dtype=mx.float32, key=None): + noise = mx.random.normal(shape, key=key) + return ( + noise * self._sigmas[-1] * (self._sigmas[-1].square() + 1).rsqrt() + ).astype(dtype) + + def sigmas(self, t): + return _interp(self._sigmas, t) + + def timesteps(self, num_steps: int, dtype=mx.float32): + steps = _linspace(len(self._sigmas) - 1, 0, num_steps + 1).astype(dtype) + return list(zip(steps, steps[1:])) + + def step(self, eps_pred, x_t, t, t_prev): + sigma = self.sigmas(t).astype(eps_pred.dtype) + sigma_prev = self.sigmas(t_prev).astype(eps_pred.dtype) + + dt = sigma_prev - sigma + x_t_prev = (sigma.square() + 1).sqrt() * x_t + eps_pred * dt + + x_t_prev = x_t_prev * (sigma_prev.square() + 1).rsqrt() + + return x_t_prev diff --git a/backends/mlx/stable_diffusion/tokenizer.py b/backends/mlx/stable_diffusion/tokenizer.py new file mode 100644 index 0000000..ae9b967 --- /dev/null +++ b/backends/mlx/stable_diffusion/tokenizer.py @@ -0,0 +1,100 @@ +# Copyright © 2023 Apple Inc. + +import regex + + +class Tokenizer: + """A simple port of CLIPTokenizer from https://github.com/huggingface/transformers/ .""" + + def __init__(self, bpe_ranks, vocab): + self.bpe_ranks = bpe_ranks + self.vocab = vocab + self.pat = regex.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + regex.IGNORECASE, + ) + + self._cache = {self.bos: self.bos, self.eos: self.eos} + + @property + def bos(self): + return "<|startoftext|>" + + @property + def bos_token(self): + return self.vocab[self.bos] + + @property + def eos(self): + return "<|endoftext|>" + + @property + def eos_token(self): + return self.vocab[self.eos] + + def bpe(self, text): + if text in self._cache: + return self._cache[text] + + unigrams = list(text[:-1]) + [text[-1] + ""] + unique_bigrams = set(zip(unigrams, unigrams[1:])) + + if not unique_bigrams: + return unigrams + + # In every iteration try to merge the two most likely bigrams. If none + # was merged we are done. + # + # Ported from https://github.com/huggingface/transformers/blob/main/src/transformers/models/clip/tokenization_clip.py + while unique_bigrams: + bigram = min( + unique_bigrams, key=lambda pair: self.bpe_ranks.get(pair, float("inf")) + ) + if bigram not in self.bpe_ranks: + break + + new_unigrams = [] + skip = False + for a, b in zip(unigrams, unigrams[1:]): + if skip: + skip = False + continue + + if (a, b) == bigram: + new_unigrams.append(a + b) + skip = True + + else: + new_unigrams.append(a) + + if not skip: + new_unigrams.append(b) + + unigrams = new_unigrams + unique_bigrams = set(zip(unigrams, unigrams[1:])) + + self._cache[text] = unigrams + + return unigrams + + def tokenize(self, text, prepend_bos=True, append_eos=True): + if isinstance(text, list): + return [self.tokenize(t, prepend_bos, append_eos) for t in text] + + # Lower case cleanup and split according to self.pat. Hugging Face does + # a much more thorough job here but this should suffice for 95% of + # cases. + clean_text = regex.sub(r"\s+", " ", text.lower()) + tokens = regex.findall(self.pat, clean_text) + + # Split the tokens according to the byte-pair merge file + bpe_tokens = [ti for t in tokens for ti in self.bpe(t)] + + # Map to token ids and return + tokens = [self.vocab[t] for t in bpe_tokens] + if prepend_bos: + tokens = [self.bos_token] + tokens + if append_eos: + tokens.append(self.eos_token) + + return tokens diff --git a/backends/mlx/stable_diffusion/unet.py b/backends/mlx/stable_diffusion/unet.py new file mode 100644 index 0000000..c1a3121 --- /dev/null +++ b/backends/mlx/stable_diffusion/unet.py @@ -0,0 +1,425 @@ +# Copyright © 2023 Apple Inc. + +import math +from typing import Optional + +import mlx.core as mx +import mlx.nn as nn + +from .config import UNetConfig + + +def upsample_nearest(x, scale: int = 2): + B, H, W, C = x.shape + x = mx.broadcast_to(x[:, :, None, :, None, :], (B, H, scale, W, scale, C)) + x = x.reshape(B, H * scale, W * scale, C) + + return x + + +class TimestepEmbedding(nn.Module): + def __init__(self, in_channels: int, time_embed_dim: int): + super().__init__() + + self.linear_1 = nn.Linear(in_channels, time_embed_dim) + self.linear_2 = nn.Linear(time_embed_dim, time_embed_dim) + + def __call__(self, x): + x = self.linear_1(x) + x = nn.silu(x) + x = self.linear_2(x) + + return x + + +class TransformerBlock(nn.Module): + def __init__( + self, + model_dims: int, + num_heads: int, + hidden_dims: Optional[int] = None, + memory_dims: Optional[int] = None, + ): + super().__init__() + + self.norm1 = nn.LayerNorm(model_dims) + self.attn1 = nn.MultiHeadAttention(model_dims, num_heads) + self.attn1.out_proj.bias = mx.zeros(model_dims) + + memory_dims = memory_dims or model_dims + self.norm2 = nn.LayerNorm(model_dims) + self.attn2 = nn.MultiHeadAttention( + model_dims, num_heads, key_input_dims=memory_dims + ) + self.attn2.out_proj.bias = mx.zeros(model_dims) + + hidden_dims = hidden_dims or 4 * model_dims + self.norm3 = nn.LayerNorm(model_dims) + self.linear1 = nn.Linear(model_dims, hidden_dims) + self.linear2 = nn.Linear(model_dims, hidden_dims) + self.linear3 = nn.Linear(hidden_dims, model_dims) + + def __call__(self, x, memory, attn_mask, memory_mask): + # Self attention + y = self.norm1(x) + y = self.attn1(y, y, y, attn_mask) + x = x + y + + # Cross attention + y = self.norm2(x) + y = self.attn2(y, memory, memory, memory_mask) + x = x + y + + # FFN + y = self.norm3(x) + y_a = self.linear1(y) + y_b = self.linear2(y) + y = y_a * nn.gelu_approx(y_b) # approximate gelu? + y = self.linear3(y) + x = x + y + + return x + + +class Transformer2D(nn.Module): + """A transformer model for inputs with 2 spatial dimensions.""" + + def __init__( + self, + in_channels: int, + model_dims: int, + encoder_dims: int, + num_heads: int, + num_layers: int = 1, + norm_num_groups: int = 32, + ): + super().__init__() + + self.norm = nn.GroupNorm(norm_num_groups, in_channels, pytorch_compatible=True) + self.proj_in = nn.Linear(in_channels, model_dims) + self.transformer_blocks = [ + TransformerBlock(model_dims, num_heads, memory_dims=encoder_dims) + for i in range(num_layers) + ] + self.proj_out = nn.Linear(model_dims, in_channels) + + def __call__(self, x, encoder_x, attn_mask, encoder_attn_mask): + # Save the input to add to the output + input_x = x + + # Perform the input norm and projection + B, H, W, C = x.shape + x = self.norm(x).reshape(B, -1, C) + x = self.proj_in(x) + + # Apply the transformer + for block in self.transformer_blocks: + x = block(x, encoder_x, attn_mask, encoder_attn_mask) + + # Apply the output projection and reshape + x = self.proj_out(x) + x = x.reshape(B, H, W, C) + + return x + input_x + + +class ResnetBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + groups: int = 32, + temb_channels: Optional[int] = None, + ): + super().__init__() + + out_channels = out_channels or in_channels + + self.norm1 = nn.GroupNorm(groups, in_channels, pytorch_compatible=True) + self.conv1 = nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + if temb_channels is not None: + self.time_emb_proj = nn.Linear(temb_channels, out_channels) + self.norm2 = nn.GroupNorm(groups, out_channels, pytorch_compatible=True) + self.conv2 = nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + + if in_channels != out_channels: + self.conv_shortcut = nn.Linear(in_channels, out_channels) + + def __call__(self, x, temb=None): + if temb is not None: + temb = self.time_emb_proj(nn.silu(temb)) + + y = self.norm1(x) + y = nn.silu(y) + y = self.conv1(y) + if temb is not None: + y = y + temb[:, None, None, :] + y = self.norm2(y) + y = nn.silu(y) + y = self.conv2(y) + + x = y + (x if "conv_shortcut" not in self else self.conv_shortcut(x)) + + return x + + +class UNetBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + prev_out_channels: Optional[int] = None, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + num_attention_heads: int = 8, + cross_attention_dim=1280, + resnet_groups: int = 32, + add_downsample=True, + add_upsample=True, + add_cross_attention=True, + ): + super().__init__() + + # Prepare the in channels list for the resnets + if prev_out_channels is None: + in_channels_list = [in_channels] + [out_channels] * (num_layers - 1) + else: + in_channels_list = [prev_out_channels] + [out_channels] * (num_layers - 1) + res_channels_list = [out_channels] * (num_layers - 1) + [in_channels] + in_channels_list = [ + a + b for a, b in zip(in_channels_list, res_channels_list) + ] + + # Add resnet blocks that also process the time embedding + self.resnets = [ + ResnetBlock2D( + in_channels=ic, + out_channels=out_channels, + temb_channels=temb_channels, + groups=resnet_groups, + ) + for ic in in_channels_list + ] + + # Add optional cross attention layers + if add_cross_attention: + self.attentions = [ + Transformer2D( + in_channels=out_channels, + model_dims=out_channels, + num_heads=num_attention_heads, + num_layers=transformer_layers_per_block, + encoder_dims=cross_attention_dim, + ) + for i in range(num_layers) + ] + + # Add an optional downsampling layer + if add_downsample: + self.downsample = nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=2, padding=1 + ) + + # or upsampling layer + if add_upsample: + self.upsample = nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + + def __call__( + self, + x, + encoder_x=None, + temb=None, + attn_mask=None, + encoder_attn_mask=None, + residual_hidden_states=None, + ): + output_states = [] + + for i in range(len(self.resnets)): + if residual_hidden_states is not None: + x = mx.concatenate([x, residual_hidden_states.pop()], axis=-1) + + x = self.resnets[i](x, temb) + + if "attentions" in self: + x = self.attentions[i](x, encoder_x, attn_mask, encoder_attn_mask) + + output_states.append(x) + + if "downsample" in self: + x = self.downsample(x) + output_states.append(x) + + if "upsample" in self: + x = self.upsample(upsample_nearest(x)) + output_states.append(x) + + return x, output_states + + +class UNetModel(nn.Module): + """The conditional 2D UNet model that actually performs the denoising.""" + + def __init__(self, config: UNetConfig): + super().__init__() + + self.conv_in = nn.Conv2d( + config.in_channels, + config.block_out_channels[0], + config.conv_in_kernel, + padding=(config.conv_in_kernel - 1) // 2, + ) + + self.timesteps = nn.SinusoidalPositionalEncoding( + config.block_out_channels[0], + max_freq=1, + min_freq=math.exp( + -math.log(10000) + 2 * math.log(10000) / config.block_out_channels[0] + ), + scale=1.0, + cos_first=True, + full_turns=False, + ) + self.time_embedding = TimestepEmbedding( + config.block_out_channels[0], + config.block_out_channels[0] * 4, + ) + + # Make the downsampling blocks + block_channels = [config.block_out_channels[0]] + list( + config.block_out_channels + ) + self.down_blocks = [ + UNetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=config.block_out_channels[0] * 4, + num_layers=config.layers_per_block[i], + transformer_layers_per_block=config.transformer_layers_per_block[i], + num_attention_heads=config.num_attention_heads[i], + cross_attention_dim=config.cross_attention_dim[i], + resnet_groups=config.norm_num_groups, + add_downsample=(i < len(config.block_out_channels) - 1), + add_upsample=False, + add_cross_attention=(i < len(config.block_out_channels) - 1), + ) + for i, (in_channels, out_channels) in enumerate( + zip(block_channels, block_channels[1:]) + ) + ] + + # Make the middle block + self.mid_blocks = [ + ResnetBlock2D( + in_channels=config.block_out_channels[-1], + out_channels=config.block_out_channels[-1], + temb_channels=config.block_out_channels[0] * 4, + groups=config.norm_num_groups, + ), + Transformer2D( + in_channels=config.block_out_channels[-1], + model_dims=config.block_out_channels[-1], + num_heads=config.num_attention_heads[-1], + num_layers=config.transformer_layers_per_block[-1], + encoder_dims=config.cross_attention_dim[-1], + ), + ResnetBlock2D( + in_channels=config.block_out_channels[-1], + out_channels=config.block_out_channels[-1], + temb_channels=config.block_out_channels[0] * 4, + groups=config.norm_num_groups, + ), + ] + + # Make the upsampling blocks + block_channels = ( + [config.block_out_channels[0]] + + list(config.block_out_channels) + + [config.block_out_channels[-1]] + ) + self.up_blocks = [ + UNetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=config.block_out_channels[0] * 4, + prev_out_channels=prev_out_channels, + num_layers=config.layers_per_block[i] + 1, + transformer_layers_per_block=config.transformer_layers_per_block[i], + num_attention_heads=config.num_attention_heads[i], + cross_attention_dim=config.cross_attention_dim[i], + resnet_groups=config.norm_num_groups, + add_downsample=False, + add_upsample=(i > 0), + add_cross_attention=(i < len(config.block_out_channels) - 1), + ) + for i, (in_channels, out_channels, prev_out_channels) in reversed( + list( + enumerate( + zip(block_channels, block_channels[1:], block_channels[2:]) + ) + ) + ) + ] + + self.conv_norm_out = nn.GroupNorm( + config.norm_num_groups, + config.block_out_channels[0], + pytorch_compatible=True, + ) + self.conv_out = nn.Conv2d( + config.block_out_channels[0], + config.out_channels, + config.conv_out_kernel, + padding=(config.conv_out_kernel - 1) // 2, + ) + + def __call__(self, x, timestep, encoder_x, attn_mask=None, encoder_attn_mask=None): + + # Compute the time embeddings + temb = self.timesteps(timestep).astype(x.dtype) + temb = self.time_embedding(temb) + + # Preprocess the input + x = self.conv_in(x) + + # Run the downsampling part of the unet + residuals = [x] + for block in self.down_blocks: + x, res = block( + x, + encoder_x=encoder_x, + temb=temb, + attn_mask=attn_mask, + encoder_attn_mask=encoder_attn_mask, + ) + residuals.extend(res) + + # Run the middle part of the unet + x = self.mid_blocks[0](x, temb) + x = self.mid_blocks[1](x, encoder_x, attn_mask, encoder_attn_mask) + x = self.mid_blocks[2](x, temb) + + # Run the upsampling part of the unet + for block in self.up_blocks: + x, _ = block( + x, + encoder_x=encoder_x, + temb=temb, + attn_mask=attn_mask, + encoder_attn_mask=encoder_attn_mask, + residual_hidden_states=residuals, + ) + + # Postprocess the output + x = self.conv_norm_out(x) + x = nn.silu(x) + x = self.conv_out(x) + + return x diff --git a/backends/mlx/stable_diffusion/vae.py b/backends/mlx/stable_diffusion/vae.py new file mode 100644 index 0000000..fe473d4 --- /dev/null +++ b/backends/mlx/stable_diffusion/vae.py @@ -0,0 +1,268 @@ +# Copyright © 2023 Apple Inc. + +import math +from typing import List + +import mlx.core as mx +import mlx.nn as nn + +from .config import AutoencoderConfig +from .unet import ResnetBlock2D, upsample_nearest + + +class Attention(nn.Module): + """A single head unmasked attention for use with the VAE.""" + + def __init__(self, dims: int, norm_groups: int = 32): + super().__init__() + + self.group_norm = nn.GroupNorm(norm_groups, dims, pytorch_compatible=True) + self.query_proj = nn.Linear(dims, dims) + self.key_proj = nn.Linear(dims, dims) + self.value_proj = nn.Linear(dims, dims) + self.out_proj = nn.Linear(dims, dims) + + def __call__(self, x): + B, H, W, C = x.shape + + y = self.group_norm(x) + + queries = self.query_proj(y).reshape(B, H * W, C) + keys = self.key_proj(y).reshape(B, H * W, C) + values = self.value_proj(y).reshape(B, H * W, C) + + scale = 1 / math.sqrt(queries.shape[-1]) + scores = (queries * scale) @ keys.transpose(0, 2, 1) + attn = mx.softmax(scores, axis=-1) + y = (attn @ values).reshape(B, H, W, C) + + y = self.out_proj(y) + x = x + y + + return x + + +class EncoderDecoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + num_layers: int = 1, + resnet_groups: int = 32, + add_downsample=True, + add_upsample=True, + ): + super().__init__() + + # Add the resnet blocks + self.resnets = [ + ResnetBlock2D( + in_channels=in_channels if i == 0 else out_channels, + out_channels=out_channels, + groups=resnet_groups, + ) + for i in range(num_layers) + ] + + # Add an optional downsampling layer + if add_downsample: + self.downsample = nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=2, padding=1 + ) + + # or upsampling layer + if add_upsample: + self.upsample = nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + + def __call__(self, x): + for resnet in self.resnets: + x = resnet(x) + + if "downsample" in self: + x = self.downsample(x) + + if "upsample" in self: + x = self.upsample(upsample_nearest(x)) + + return x + + +class Encoder(nn.Module): + """Implements the encoder side of the Autoencoder.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + block_out_channels: List[int] = [64], + layers_per_block: int = 2, + resnet_groups: int = 32, + ): + super().__init__() + + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1 + ) + + channels = [block_out_channels[0]] + list(block_out_channels) + self.down_blocks = [ + EncoderDecoderBlock2D( + in_channels, + out_channels, + num_layers=layers_per_block, + resnet_groups=resnet_groups, + add_downsample=i < len(block_out_channels) - 1, + add_upsample=False, + ) + for i, (in_channels, out_channels) in enumerate(zip(channels, channels[1:])) + ] + + self.mid_blocks = [ + ResnetBlock2D( + in_channels=block_out_channels[-1], + out_channels=block_out_channels[-1], + groups=resnet_groups, + ), + Attention(block_out_channels[-1], resnet_groups), + ResnetBlock2D( + in_channels=block_out_channels[-1], + out_channels=block_out_channels[-1], + groups=resnet_groups, + ), + ] + + self.conv_norm_out = nn.GroupNorm( + resnet_groups, block_out_channels[-1], pytorch_compatible=True + ) + self.conv_out = nn.Conv2d(block_out_channels[-1], out_channels, 3, padding=1) + + def __call__(self, x): + x = self.conv_in(x) + + for l in self.down_blocks: + x = l(x) + + x = self.mid_blocks[0](x) + x = self.mid_blocks[1](x) + x = self.mid_blocks[2](x) + + x = self.conv_norm_out(x) + x = nn.silu(x) + x = self.conv_out(x) + + return x + + +class Decoder(nn.Module): + """Implements the decoder side of the Autoencoder.""" + + def __init__( + self, + in_channels: int, + out_channels: int, + block_out_channels: List[int] = [64], + layers_per_block: int = 2, + resnet_groups: int = 32, + ): + super().__init__() + + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1 + ) + + self.mid_blocks = [ + ResnetBlock2D( + in_channels=block_out_channels[-1], + out_channels=block_out_channels[-1], + groups=resnet_groups, + ), + Attention(block_out_channels[-1], resnet_groups), + ResnetBlock2D( + in_channels=block_out_channels[-1], + out_channels=block_out_channels[-1], + groups=resnet_groups, + ), + ] + + channels = list(reversed(block_out_channels)) + channels = [channels[0]] + channels + self.up_blocks = [ + EncoderDecoderBlock2D( + in_channels, + out_channels, + num_layers=layers_per_block, + resnet_groups=resnet_groups, + add_downsample=False, + add_upsample=i < len(block_out_channels) - 1, + ) + for i, (in_channels, out_channels) in enumerate(zip(channels, channels[1:])) + ] + + self.conv_norm_out = nn.GroupNorm( + resnet_groups, block_out_channels[0], pytorch_compatible=True + ) + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1) + + def __call__(self, x): + x = self.conv_in(x) + + x = self.mid_blocks[0](x) + x = self.mid_blocks[1](x) + x = self.mid_blocks[2](x) + + for l in self.up_blocks: + x = l(x) + + x = self.conv_norm_out(x) + x = nn.silu(x) + x = self.conv_out(x) + + return x + + +class Autoencoder(nn.Module): + """The autoencoder that allows us to perform diffusion in the latent space.""" + + def __init__(self, config: AutoencoderConfig): + super().__init__() + + self.latent_channels = config.latent_channels_in + self.scaling_factor = config.scaling_factor + self.encoder = Encoder( + config.in_channels, + config.latent_channels_out, + config.block_out_channels, + config.layers_per_block, + resnet_groups=config.norm_num_groups, + ) + self.decoder = Decoder( + config.latent_channels_in, + config.out_channels, + config.block_out_channels, + config.layers_per_block + 1, + resnet_groups=config.norm_num_groups, + ) + + self.quant_proj = nn.Linear( + config.latent_channels_out, config.latent_channels_out + ) + self.post_quant_proj = nn.Linear( + config.latent_channels_in, config.latent_channels_in + ) + + def decode(self, z): + return self.decoder(self.post_quant_proj(z)) + + def __call__(self, x, key=None): + x = self.encoder(x) + x = self.quant_proj(x) + + mean, logvar = x.split(2, axis=-1) + std = mx.exp(0.5 * logvar) + z = mx.random.normal(mean.shape, key=key) * std + mean + + x_hat = self.decode(z) + + return dict(x_hat=x_hat, z=z, mean=mean, logvar=logvar) diff --git a/main.py b/main.py index 2d71129..5157144 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,8 @@ import dotenv from nostr_dvm.bot import Bot from nostr_dvm.tasks import videogeneration_replicate_svd, imagegeneration_replicate_sdxl, textgeneration_llmlite, \ trending_notes_nostrband, discovery_inactive_follows, translation_google, textextraction_pdf, \ - translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle, texttospeech + translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle, texttospeech, \ + imagegeneration_mlx, advanced_search, textextraction_whisper_mlx from nostr_dvm.utils.admin_utils import AdminConfig from nostr_dvm.utils.backend_utils import keep_alive from nostr_dvm.utils.definitions import EventDefinitions @@ -138,6 +139,14 @@ def playground(): bot_config.SUPPORTED_DVMS.append(tts) tts.run() + from sys import platform + if platform == "darwin": + # Test with MLX for OSX M1/M2/M3 chips + mlx = imagegeneration_mlx.build_example("SD with MLX", "mlx_sd", admin_config) + bot_config.SUPPORTED_DVMS.append(mlx) + mlx.run() + + # Run the bot Bot(bot_config) # Keep the main function alive for libraries that require it, like openai diff --git a/nostr_dvm/tasks/imagegeneration_mlx.py b/nostr_dvm/tasks/imagegeneration_mlx.py new file mode 100644 index 0000000..8743bb9 --- /dev/null +++ b/nostr_dvm/tasks/imagegeneration_mlx.py @@ -0,0 +1,186 @@ +import json +import os +from PIL import Image +from tqdm import tqdm + +from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface +from nostr_dvm.utils.admin_utils import AdminConfig +from nostr_dvm.utils.definitions import EventDefinitions +from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config +from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag +from nostr_dvm.utils.output_utils import upload_media_to_hoster +from nostr_dvm.utils.zap_utils import get_price_per_sat + +""" +This File contains a Module to generate an Image on replicate and receive results back. + +Accepted Inputs: Prompt (text) +Outputs: An url to an Image +Params: +""" + + +class ImageGenerationMLX(DVMTaskInterface): + KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE + TASK: str = "text-to-image" + FIX_COST: float = 120 + dependencies = [("nostr-dvm", "nostr-dvm"), + ("mlx", "mlx"), + ("safetensors", "safetensors"), + ("huggingface-hub", "huggingface-hub"), + ("regex", "regex"), + ("tqdm", "tqdm"), + ] + + def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, + admin_config: AdminConfig = None, options=None): + dvm_config.SCRIPT = os.path.abspath(__file__) + super().__init__(name, dvm_config, nip89config, admin_config, options) + + def is_input_supported(self, tags): + for tag in tags: + if tag.as_vec()[0] == 'i': + input_value = tag.as_vec()[1] + input_type = tag.as_vec()[2] + if input_type != "text": + return False + + elif tag.as_vec()[0] == 'output': + output = tag.as_vec()[1] + if (output == "" or + not (output == "image/png" or "image/jpg" + or output == "image/png;format=url" or output == "image/jpg;format=url")): + print("Output format not supported, skipping..") + return False + + return True + + def create_request_from_nostr_event(self, event, client=None, dvm_config=None): + request_form = {"jobID": event.id().to_hex() + "_" + self.NAME.replace(" ", "")} + prompt = "" + width = "1024" + height = "1024" + + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + input_type = tag.as_vec()[2] + if input_type == "text": + prompt = tag.as_vec()[1] + + elif tag.as_vec()[0] == 'param': + print("Param: " + tag.as_vec()[1] + ": " + tag.as_vec()[2]) + if tag.as_vec()[1] == "size": + if len(tag.as_vec()) > 3: + width = (tag.as_vec()[2]) + height = (tag.as_vec()[3]) + elif len(tag.as_vec()) == 3: + split = tag.as_vec()[2].split("x") + if len(split) > 1: + width = split[0] + height = split[1] + elif tag.as_vec()[1] == "model": + model = tag.as_vec()[2] + elif tag.as_vec()[1] == "quality": + quality = tag.as_vec()[2] + + options = { + "prompt": prompt, + "size": width + "x" + height, + "number": 1 + } + request_form['options'] = json.dumps(options) + + return request_form + + def process(self, request_form): + try: + import mlx.core as mx + from backends.mlx.stable_diffusion import StableDiffusion + options = DVMTaskInterface.set_options(request_form) + + sd = StableDiffusion() + cfg_weight = 7.5 + batchsize = 1 + n_rows = 1 + steps = 50 + n_images = options["number"] + + # Generate the latent vectors using diffusion + latents = sd.generate_latents( + options["prompt"], + n_images=n_images, + cfg_weight=cfg_weight, + num_steps=steps, + negative_text="", + ) + for x_t in tqdm(latents, total=steps): + mx.simplify(x_t) + mx.simplify(x_t) + mx.eval(x_t) + + # Decode them into images + decoded = [] + for i in tqdm(range(0, 1, batchsize)): + decoded.append(sd.decode(x_t[i: i + batchsize])) + mx.eval(decoded[-1]) + + # Arrange them on a grid + x = mx.concatenate(decoded, axis=0) + x = mx.pad(x, [(0, 0), (8, 8), (8, 8), (0, 0)]) + B, H, W, C = x.shape + x = x.reshape(n_rows, B // n_rows, H, W, C).transpose(0, 2, 1, 3, 4) + x = x.reshape(n_rows * H, B // n_rows * W, C) + x = (x * 255).astype(mx.uint8) + + # Save them to disc + image = Image.fromarray(x.__array__()) + image.save("./outputs/image.jpg") + result = upload_media_to_hoster("./outputs/image.jpg") + return result + + except Exception as e: + print("Error in Module") + raise Exception(e) + + +# We build an example here that we can call by either calling this file directly from the main directory, +# or by adding it to our playground. You can call the example and adjust it to your needs or redefine it in the +# playground or elsewhere +def build_example(name, identifier, admin_config): + dvm_config = build_default_config(identifier) + admin_config.LUD16 = dvm_config.LN_ADDRESS + profit_in_sats = 10 + dvm_config.FIX_COST = int(((4.0 / (get_price_per_sat("USD") * 100)) + profit_in_sats)) + + nip89info = { + "name": name, + "image": "https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg", + "about": "I use Replicate to run StableDiffusion XL", + "encryptionSupported": True, + "cashuAccepted": True, + "nip90Params": { + "size": { + "required": False, + "values": ["1024:1024", "1024x1792", "1792x1024"] + } + } + } + + nip89config = NIP89Config() + nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"]) + nip89config.CONTENT = json.dumps(nip89info) + + return ImageGenerationMLX(name=name, dvm_config=dvm_config, nip89config=nip89config, + admin_config=admin_config) + + +def process_venv(): + args = DVMTaskInterface.process_args() + dvm_config = build_default_config(args.identifier) + dvm = ImageGenerationMLX(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None) + result = dvm.process(json.loads(args.request)) + DVMTaskInterface.write_output(result, args.output) + + +if __name__ == '__main__': + process_venv() diff --git a/nostr_dvm/utils/zap_utils.py b/nostr_dvm/utils/zap_utils.py index c9d8680..59c3a2f 100644 --- a/nostr_dvm/utils/zap_utils.py +++ b/nostr_dvm/utils/zap_utils.py @@ -300,10 +300,6 @@ def get_price_per_sat(currency): def make_ln_address_nostdress(identifier, npub, pin, nostdressdomain): - # env_path = Path('.env') - # if env_path.is_file(): - # dotenv.load_dotenv(env_path, verbose=True, override=True) - print(os.getenv("LNBITS_INVOICE_KEY_" + identifier.upper())) data = { 'name': identifier, diff --git a/setup.py b/setup.py index 628e364..486ecd0 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '0.0.9' +VERSION = '0.1.0' DESCRIPTION = 'A framework to build and run Nostr NIP90 Data Vending Machines' LONG_DESCRIPTION = ('A framework to build and run Nostr NIP90 Data Vending Machines. ' 'This is an early stage release. Interfaces might change/brick') @@ -13,8 +13,10 @@ setup( author_email="believethehypeonnostr@proton.me", description=DESCRIPTION, long_description=LONG_DESCRIPTION, - packages=find_packages(include=['nostr_dvm', 'nostr_dvm.backends', 'nostr_dvm.interfaces', 'nostr_dvm.tasks', - 'nostr_dvm.utils', 'nostr_dvm.utils.scrapper']), + packages=find_packages(include=['nostr_dvm', 'nostr_dvm.interfaces', 'nostr_dvm.tasks', + 'nostr_dvm.utils', 'nostr_dvm.utils.scrapper', + 'nostr_dvm.backends', 'nostr_dvm.backends.mlx', + 'nostr_dvm.backends.mlx.stablediffusion']), install_requires=["nostr-sdk==0.0.5", "bech32==1.2.0", "pycryptodome==3.19.0", @@ -32,7 +34,7 @@ setup( "moviepy==2.0.0.dev2", "zipp==3.17.0", "urllib3==2.1.0", - "typing_extensions==4.8.0" + "typing_extensions>=4.9.0" ], keywords=['nostr', 'nip90', 'dvm', 'data vending machine'], url="https://github.com/believethehype/nostrdvm", diff --git a/tests/test_dvm_client.py b/tests/test_dvm_client.py index b47d414..4315487 100644 --- a/tests/test_dvm_client.py +++ b/tests/test_dvm_client.py @@ -151,8 +151,8 @@ def nostr_client(): #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) #nostr_client_test_translation("note1p8cx2dz5ss5gnk7c59zjydcncx6a754c0hsyakjvnw8xwlm5hymsnc23rs", "event", "es", 20,20) #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "zh", 20, 20) - #nostr_client_test_image("a beautiful purple ostrich watching the sunset") - nostr_client_test_tts("Hello, this is a test. Mic check one, two.") + nostr_client_test_image("a beautiful purple ostrich watching the sunset") + #nostr_client_test_tts("Hello, this is a test. Mic check one, two.") #cashutoken = "cashuAeyJ0b2tlbiI6W3sicHJvb2ZzIjpbeyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MSwiQyI6IjAyNWU3ODZhOGFkMmExYTg0N2YxMzNiNGRhM2VhMGIyYWRhZGFkOTRiYzA4M2E2NWJjYjFlOTgwYTE1NGIyMDA2NCIsInNlY3JldCI6InQ1WnphMTZKMGY4UElQZ2FKTEg4V3pPck5rUjhESWhGa291LzVzZFd4S0U9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6NCwiQyI6IjAyOTQxNmZmMTY2MzU5ZWY5ZDc3MDc2MGNjZmY0YzliNTMzMzVmZTA2ZGI5YjBiZDg2Njg5Y2ZiZTIzMjVhYWUwYiIsInNlY3JldCI6IlRPNHB5WE43WlZqaFRQbnBkQ1BldWhncm44UHdUdE5WRUNYWk9MTzZtQXM9In0seyJpZCI6InZxc1VRSVorb0sxOSIsImFtb3VudCI6MTYsIkMiOiIwMmRiZTA3ZjgwYmMzNzE0N2YyMDJkNTZiMGI3ZTIzZTdiNWNkYTBhNmI3Yjg3NDExZWYyOGRiZDg2NjAzNzBlMWIiLCJzZWNyZXQiOiJHYUNIdHhzeG9HM3J2WWNCc0N3V0YxbU1NVXczK0dDN1RKRnVwOHg1cURzPSJ9XSwibWludCI6Imh0dHBzOi8vbG5iaXRzLmJpdGNvaW5maXhlc3RoaXMub3JnL2Nhc2h1L2FwaS92MS9ScDlXZGdKZjlxck51a3M1eVQ2SG5rIn1dfQ=="