diff --git a/.env_example b/.env_example index ea271f8..a5718bf 100644 --- a/.env_example +++ b/.env_example @@ -1,9 +1,14 @@ -NOSTR_PRIVATE_KEY = nostrSecretkeyinhex -NOSTR_TEST_CLIENT_PRIVATE_KEY = nostrSecretkeyinhex_forthetestclient -USER_DB_PATH = nostrzaps.db +NOSTR_PRIVATE_KEY = "nostrSecretkeyinhex" +NOSTR_TEST_CLIENT_PRIVATE_KEY = "nostrSecretkeyinhex_forthetestclient" +USER_DB_PATH = "nostrzaps.db" -LNBITS_INVOICE_KEY = lnbitswalletinvoicekey -LNBITS_HOST = https://lnbits.com +# Optional LNBITS options to create invoices (if empty, it will use the lud16 from profile) +LNBITS_INVOICE_KEY = "" +LNBITS_HOST = "https://lnbits.com" TASK_TEXTEXTRACTION_NIP89_DTAG = "asdd" -TASK_TRANSLATION_NIP89_DTAG = abcded +TASK_TRANSLATION_NIP89_DTAG = "abcded" +TASK_IMAGEGENERATION_NIP89_DTAG = "fgdfgdf" + +#Backend Specific Options for tasks that require them +NOVA_SERVER = "127.0.0.1:37318" \ No newline at end of file diff --git a/README.md b/README.md index 637e042..db4f1e6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Nostr AI Data Vending Machine +# NostrAI Data Vending Machine This example DVM implementation in Python currently supports simple translations using Google translate, as well as extraction of text from links with pdf files. diff --git a/backends/README.md b/backends/README.md new file mode 100644 index 0000000..c7a2bac --- /dev/null +++ b/backends/README.md @@ -0,0 +1,10 @@ +# NostrAI Data Vending Machine Backends + +Each DVM task might either run locally or use a specific backend. +Especially for GPU tasks it might make sense to outsource some tasks on other machines. +Backends can also be API calls to (paid) services. This directory contains basic calling functions to such backends. +Modules in the folder "tasks" might use these functions to call a specific backend. + +Using backends might require some extra work like running/hosting a server or acquiring an API key. + + diff --git a/backends/nova_server.py b/backends/nova_server.py new file mode 100644 index 0000000..c631241 --- /dev/null +++ b/backends/nova_server.py @@ -0,0 +1,105 @@ +import io +import json +import os +import time +import zipfile +import pandas as pd +import requests +import PIL.Image as Image + +from utils.output_utils import uploadMediaToHoster + +""" +This file contains basic calling functions for ML tasks that are outsourced to nova-server +(https://github.com/hcmlab/nova-server). nova-server is an Open-Source backend that enables running models locally, by +accepting a request form. Modules are deployed in in separate virtual environments so dependencies won't conflict. + +Setup nova-server: +https://hcmlab.github.io/nova-server/docbuild/html/tutorials/introduction.html + +""" + +""" +send_request_to_nova_server(request_form) +Function to send a request_form to the server, containing all the information we parsed from the Nostr event and added +in the module that is calling the server + +""" + + +def send_request_to_nova_server(request_form, address): + print("Sending job to NOVA-Server") + url = ('http://' + address + '/' + str(request_form["mode"]).lower()) + headers = {'Content-type': 'application/x-www-form-urlencoded'} + response = requests.post(url, headers=headers, data=request_form) + return response.content + + +""" +check_nova_server_status(request_form) +Function that requests the status of the current process with the jobID (we use the Nostr event as jobID). +When the Job is successfully finished we grab the result and depending on the type return the output +We throw an exception on error +""" + + +def check_nova_server_status(jobID, address): + headers = {'Content-type': 'application/x-www-form-urlencoded'} + url_status = 'http://' + address + '/job_status' + url_log = 'http://' + address + '/log' + + print("Sending Status Request to NOVA-Server") + data = {"jobID": jobID} + + status = 0 + length = 0 + while status != 2 and status != 3: + response_status = requests.post(url_status, headers=headers, data=data) + response_log = requests.post(url_log, headers=headers, data=data) + status = int(json.loads(response_status.text)['status']) + + log = str(response_log.content)[length:] + length = len(str(response_log.content)) + if log != "": + print(log + " Status: " + str(status)) + # WAITING = 0, RUNNING = 1, FINISHED = 2, ERROR = 3 + time.sleep(1.0) + + if status == 2: + try: + result = "" + url_fetch = 'http://' + address + '/fetch_result' + print("Fetching Results from NOVA-Server...") + data = {"jobID": jobID} + response = requests.post(url_fetch, headers=headers, data=data) + content_type = response.headers['content-type'] + print(content_type) + if content_type == "image/jpeg": + image = Image.open(io.BytesIO(response.content)) + image.save("./outputs/image.jpg") + result = uploadMediaToHoster("./outputs/image.jpg") + os.remove("./outputs/image.jpg") + elif content_type == 'text/plain; charset=utf-8': + result = response.content.decode('utf-8') + elif content_type == "zip": + zf = zipfile.ZipFile(io.BytesIO(response.content), "r") + + for fileinfo in zf.infolist(): + if fileinfo.filename.endswith(".annotation~"): + try: + anno_string = zf.read(fileinfo).decode('utf-8', errors='replace') + columns = ['from', 'to', 'name', 'conf'] + result = pd.DataFrame([row.split(';') for row in anno_string.split('\n')], + columns=columns) + print(result) + with open("response.zip", "wb") as f: + f.write(response.content) + except: + zf.extractall() + + return result + except Exception as e: + print("Couldn't fetch result: " + str(e)) + + elif status == 3: + return "error" diff --git a/dvm.py b/dvm.py index c712ff1..535be35 100644 --- a/dvm.py +++ b/dvm.py @@ -22,13 +22,14 @@ jobs_on_hold_list = [] dvm_config = DVMConfig() -def dvm(config): +def DVM(config): dvm_config = config keys = Keys.from_sk_str(dvm_config.PRIVATE_KEY) pk = keys.public_key() print(f"Nostr DVM public key: {pk.to_bech32()}, Hex: {pk.to_hex()} ") - print('Supported DVM tasks: ' + ', '.join(p.TASK for p in dvm_config.SUPPORTED_TASKS)) + print('Supported DVM tasks: ' + ', '.join(p.NAME + ":" + p.TASK for p in dvm_config.SUPPORTED_TASKS)) + client = Client(keys) for relay in dvm_config.RELAY_LIST: @@ -207,6 +208,7 @@ def dvm(config): or job_event.kind() == EventDefinitions.KIND_DM): task = get_task(job_event, client=client, dvmconfig=dvm_config) + result = "" for dvm in dvm_config.SUPPORTED_TASKS: try: if task == dvm.TASK: @@ -215,7 +217,11 @@ def dvm(config): check_and_return_event(result, str(job_event.as_json()), dvm_key=dvm_config.PRIVATE_KEY) except Exception as e: + print(e) respond_to_error(e, job_event.as_json(), is_from_bot, dvm_config.PRIVATE_KEY) + return + + def check_event_has_not_unfinished_job_input(nevent, append, client, dvmconfig): task_supported, task, duration = check_task_is_supported(nevent, client, False, config=dvmconfig) @@ -246,44 +252,43 @@ def dvm(config): def send_job_status_reaction(original_event, status, is_paid=True, amount=0, client=None, content=None, config=None, key=None): dvmconfig = config - altdesc = "This is a reaction to a NIP90 DVM AI task. " + alt_description = "This is a reaction to a NIP90 DVM AI task. " task = get_task(original_event, client=client, dvmconfig=dvmconfig) if status == "processing": - altdesc = "NIP90 DVM AI task " + task + " started processing. " - reaction = altdesc + emoji.emojize(":thumbs_up:") + alt_description = "NIP90 DVM AI task " + task + " started processing. " + reaction = alt_description + emoji.emojize(":thumbs_up:") elif status == "success": - altdesc = "NIP90 DVM AI task " + task + " finished successfully. " - reaction = altdesc + emoji.emojize(":call_me_hand:") + alt_description = "NIP90 DVM AI task " + task + " finished successfully. " + reaction = alt_description + emoji.emojize(":call_me_hand:") elif status == "chain-scheduled": - altdesc = "NIP90 DVM AI task " + task + " Chain Task scheduled" - reaction = altdesc + emoji.emojize(":thumbs_up:") + alt_description = "NIP90 DVM AI task " + task + " Chain Task scheduled" + reaction = alt_description + emoji.emojize(":thumbs_up:") elif status == "error": - altdesc = "NIP90 DVM AI task " + task + " had an error. " + alt_description = "NIP90 DVM AI task " + task + " had an error. " if content is None: - reaction = altdesc + emoji.emojize(":thumbs_down:") + reaction = alt_description + emoji.emojize(":thumbs_down:") else: - reaction = altdesc + emoji.emojize(":thumbs_down:") + content + reaction = alt_description + emoji.emojize(":thumbs_down:") + content elif status == "payment-required": - altdesc = "NIP90 DVM AI task " + task + " requires payment of min " + str(amount) + " Sats. " - reaction = altdesc + emoji.emojize(":orange_heart:") + alt_description = "NIP90 DVM AI task " + task + " requires payment of min " + str(amount) + " Sats. " + reaction = alt_description + emoji.emojize(":orange_heart:") elif status == "payment-rejected": - altdesc = "NIP90 DVM AI task " + task + " payment is below required amount of " + str(amount) + " Sats. " - reaction = altdesc + emoji.emojize(":thumbs_down:") + alt_description = "NIP90 DVM AI task " + task + " payment is below required amount of " + str(amount) + " Sats. " + reaction = alt_description + emoji.emojize(":thumbs_down:") elif status == "user-blocked-from-service": - - altdesc = "NIP90 DVM AI task " + task + " can't be performed. User has been blocked from Service. " - reaction = altdesc + emoji.emojize(":thumbs_down:") + alt_description = "NIP90 DVM AI task " + task + " can't be performed. User has been blocked from Service. " + reaction = alt_description + emoji.emojize(":thumbs_down:") else: reaction = emoji.emojize(":thumbs_down:") - etag = Tag.parse(["e", original_event.id().to_hex()]) - ptag = Tag.parse(["p", original_event.pubkey().to_hex()]) - alttag = Tag.parse(["alt", altdesc]) - statustag = Tag.parse(["status", status]) - tags = [etag, ptag, alttag, statustag] + e_tag = Tag.parse(["e", original_event.id().to_hex()]) + p_tag = Tag.parse(["p", original_event.pubkey().to_hex()]) + alt_tag = Tag.parse(["alt", alt_description]) + status_tag = Tag.parse(["status", status]) + tags = [e_tag, p_tag, alt_tag, status_tag] if status == "success" or status == "error": # for x in job_list: @@ -354,8 +359,13 @@ def dvm(config): send_nostr_reply_event(data, original_event_str, key=keys) break - post_processed_content = post_process_result(data, original_event) - send_nostr_reply_event(post_processed_content, original_event_str, key=keys) + + try: + post_processed_content = post_process_result(data, original_event) + send_nostr_reply_event(post_processed_content, original_event_str, key=keys) + except Exception as e: + respond_to_error(e, original_event_str, False, dvm_config.PRIVATE_KEY) + def send_nostr_reply_event(content, original_event_as_str, key=None): originalevent = Event.from_json(original_event_as_str) @@ -395,7 +405,7 @@ def dvm(config): sender = "" task = "" if not is_from_bot: - send_job_status_reaction(original_event, "error", content=content, key=dvm_key) + send_job_status_reaction(original_event, "error", content=str(content), key=dvm_key) # TODO Send Zap back else: for tag in original_event.tags(): @@ -407,7 +417,8 @@ def dvm(config): user = get_from_sql_table(sender) if not user.iswhitelisted: amount = int(user.balance) + get_amount_per_task(task, dvm_config) - update_sql_table(sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16, user.name, + update_sql_table(sender, amount, user.iswhitelisted, user.isblacklisted, user.nip05, user.lud16, + user.name, Timestamp.now().as_secs()) message = "There was the following error : " + content + ". Credits have been reimbursed" else: diff --git a/interfaces/dvmtaskinterface.py b/interfaces/dvmtaskinterface.py index 25316c0..d08c79e 100644 --- a/interfaces/dvmtaskinterface.py +++ b/interfaces/dvmtaskinterface.py @@ -3,7 +3,7 @@ class DVMTaskInterface: TASK: str COST: int - def NIP89_announcement(self): + def NIP89_announcement(self, d_tag, content): """Define the NIP89 Announcement""" pass diff --git a/main.py b/main.py index b8a12f1..afcab5e 100644 --- a/main.py +++ b/main.py @@ -4,36 +4,74 @@ from threading import Thread import dotenv import utils.env as env -from tasks.textextractionPDF import TextExtractionPDF +from tasks.imagegenerationsdxl import ImageGenerationSDXL +from tasks.textextractionpdf import TextExtractionPDF from tasks.translation import Translation -from utils.definitions import EventDefinitions - def run_nostr_dvm_with_local_config(): - from dvm import dvm, DVMConfig + from dvm import DVM, DVMConfig - dvmconfig = DVMConfig() - dvmconfig.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY) + dvm_config = DVMConfig() + dvm_config.PRIVATE_KEY = os.getenv(env.NOSTR_PRIVATE_KEY) - #Spawn two DVMs - PDFextactor = TextExtractionPDF("PDF Extractor", env.NOSTR_PRIVATE_KEY) - Translator = Translation("Translator", env.NOSTR_PRIVATE_KEY) + # Spawn the DVMs + # Add NIP89 events for each DVM (set rebroad_cast = True for the next start in admin_utils) + # Add the dtag here or in your .env file, so you can update your dvm later and change the content as needed. + # Get a dtag and the content at vendata.io - #Add the 2 DVMS to the config - dvmconfig.SUPPORTED_TASKS = [PDFextactor, Translator] + # Spawn DVM1 Kind 5000 Text Ectractor from PDFs + pdfextactor = TextExtractionPDF("PDF Extractor", os.getenv(env.NOSTR_PRIVATE_KEY)) + d_tag = os.getenv(env.TASK_TEXTEXTRACTION_NIP89_DTAG) + content = "{\"name\":\"" + pdfextactor.NAME + ("\",\"image\":\"https://image.nostr.build" + "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669" + ".jpg\",\"about\":\"I extract Text from pdf documents\"," + "\"nip90Params\":{}}") + dvm_config.NIP89s.append(pdfextactor.NIP89_announcement(d_tag, content)) - # Add NIP89 events for both DVMs (set rebroad_cast = True in admin_utils) - # Add the dtag in your .env file so you can update your dvm later and change the content in the module file as needed. - # Get a dtag at vendata.io - dvmconfig.NIP89s.append(PDFextactor.NIP89_announcement()) - dvmconfig.NIP89s.append(Translator.NIP89_announcement()) + # Spawn DVM2 Kind 5002 Text Translation + translator = Translation("Translator", os.getenv(env.NOSTR_PRIVATE_KEY)) + d_tag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG) + content = "{\"name\":\"" + translator.NAME + ("\",\"image\":\"https://image.nostr.build" + "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669" + ".jpg\",\"about\":\"I translate text from given text/event/job, " + "currently using Google Translation Services into language defined " + "in param. \",\"nip90Params\":{\"language\":{\"required\":true," + "\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\"," + "\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\"," + "\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\"," + "\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\"," + "\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\"," + "\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\"," + "\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\"," + "\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\"," + "\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\"," + "\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\"," + "\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\"," + "\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}") + dvm_config.NIP89s.append(translator.NIP89_announcement(d_tag, content)) - #SET Lnbits Invoice Key and Server if DVM should provide invoices directly, else make sure you have a lnaddress on the profile - dvmconfig.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY) - dvmconfig.LNBITS_URL = os.getenv(env.LNBITS_HOST) + # Spawn DVM3 Kind 5100 Image Generation This one uses a specific backend called nova-server. If you want to use + # it see the instructions in backends/nova_server + artist = ImageGenerationSDXL("Unstable Diffusion", os.getenv(env.NOSTR_PRIVATE_KEY)) + d_tag = os.getenv(env.TASK_IMAGEGENERATION_NIP89_DTAG) + content = "{\"name\":\"" + artist.NAME + ("\",\"image\":\"https://image.nostr.build" + "/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg" + "\",\"about\":\"I draw images based on a prompt with Stable Diffusion " + "XL 1.0.\",\"nip90Params\":{}}") + dvm_config.NIP89s.append(artist.NIP89_announcement(d_tag, content)) - #Start the DVM - nostr_dvm_thread = Thread(target=dvm, args=[dvmconfig]) + + + # Add the DVMS you want to use to the config + dvm_config.SUPPORTED_TASKS = [pdfextactor, translator, artist] + + # SET Lnbits Invoice Key and Server if DVM should provide invoices directly, else make sure you have a lnaddress + # on the profile + dvm_config.LNBITS_INVOICE_KEY = os.getenv(env.LNBITS_INVOICE_KEY) + dvm_config.LNBITS_URL = os.getenv(env.LNBITS_HOST) + + # Start the Server + nostr_dvm_thread = Thread(target=DVM, args=[dvm_config]) nostr_dvm_thread.start() diff --git a/requirements.txt b/requirements.txt index 16bad4f..a1df8ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ python-dateutil==2.8.2 python-dotenv==1.0.0 python-editor==1.0.4 pytz==2023.3.post1 +PyUpload~=0.1.4 pyuseragents==1.0.5 readchar==4.0.5 requests==2.31.0 @@ -31,3 +32,4 @@ translatepy==2.3 tzdata==2023.3 urllib3==2.1.0 wcwidth==0.2.10 + diff --git a/tasks/README.md b/tasks/README.md new file mode 100644 index 0000000..7dcf575 --- /dev/null +++ b/tasks/README.md @@ -0,0 +1,13 @@ +# NostrAI Data Vending Machine Tasks + +Here Tasks can be defined. Tasks need to follow the DVMTaskInterface as defined in interfaces. +Tasks can either happen locally (especially if they are fast) or they can call an alternative backend. +Reusable backend functions can be defined in backends (e.g. API calls) + +Current List of Tasks: + +| Module | Kind | Description | Backend | +|---------------------|------|-------------------------------------------|---------------------------| +| Translation | 5002 | Translates Inputs to another language | Local, calling Google API | +| TextExtractionPDF | 5001 | Extracts Text from a PDF file | Local | +| ImageGenerationSDXL | 5100 | Generates an Image with StableDiffusionXL | nova-server | \ No newline at end of file diff --git a/tasks/imagegenerationsdxl.py b/tasks/imagegenerationsdxl.py new file mode 100644 index 0000000..58ecb40 --- /dev/null +++ b/tasks/imagegenerationsdxl.py @@ -0,0 +1,120 @@ +import os +from multiprocessing.pool import ThreadPool +from backends.nova_server import check_nova_server_status, send_request_to_nova_server +from interfaces.dvmtaskinterface import DVMTaskInterface +from utils.definitions import EventDefinitions +from utils.nip89_utils import NIP89Announcement + + +""" +This File contains a Module to transform Text input on NOVA-Server and receive results back. + +Accepted Inputs: Prompt (text) +Outputs: An url to an Image +""" + + +class ImageGenerationSDXL(DVMTaskInterface): + KIND: int = EventDefinitions.KIND_NIP90_GENERATE_IMAGE + TASK: str = "text-to-image" + COST: int = 0 + + def __init__(self, name, pk): + self.NAME = name + self.PK = pk + + def NIP89_announcement(self, d_tag, content): + nip89 = NIP89Announcement() + nip89.kind = self.KIND + nip89.pk = self.PK + nip89.dtag = d_tag + nip89.content = content + return nip89 + + def is_input_supported(self, input_type, input_content): + if input_type != "text": + return False + return True + + def create_request_form_from_nostr_event(self, event, client=None, dvm_config=None): + request_form = {"jobID": event.id().to_hex() + "_"+ self.NAME.replace(" ", "")} + request_form["mode"] = "PROCESS" + request_form["trainerFilePath"] = 'modules\\stablediffusionxl\\stablediffusionxl.trainer' + + prompt = "" + negative_prompt = "" + #model = "stabilityai/stable-diffusion-xl-base-1.0" + model = "unstable" + # models: juggernautXL, dynavisionXL, colossusProjectXL, newrealityXL, unstable + ratio_width = "1" + ratio_height = "1" + width = "" + height = "" + lora = "" + lora_weight = "" + strength = "" + guidance_scale = "" + for tag in event.tags(): + if tag.as_vec()[0] == 'i': + input_type = tag.as_vec()[2] + if input_type == "text": + prompt = tag.as_vec()[1] + + elif tag.as_vec()[0] == 'param': + print(tag.as_vec()[2]) + if tag.as_vec()[1] == "negative_prompt": + negative_prompt = tag.as_vec()[2] + elif tag.as_vec()[1] == "lora": + lora = tag.as_vec()[2] + elif tag.as_vec()[1] == "lora_weight": + lora_weight = tag.as_vec()[2] + elif tag.as_vec()[1] == "strength": + strength = tag.as_vec()[2] + elif tag.as_vec()[1] == "guidance_scale": + guidance_scale = tag.as_vec()[2] + elif tag.as_vec()[1] == "ratio": + if len(tag.as_vec()) > 3: + ratio_width = (tag.as_vec()[2]) + ratio_height = (tag.as_vec()[3]) + elif len(tag.as_vec()) == 3: + split = tag.as_vec()[2].split(":") + ratio_width = split[0] + ratio_height = split[1] + #if size is set it will overwrite ratio. + elif tag.as_vec()[1] == "size": + + if len(tag.as_vec()) > 3: + width = (tag.as_vec()[2]) + height = (tag.as_vec()[3]) + elif len(tag.as_vec()) == 3: + split = tag.as_vec()[2].split("x") + if len(split) > 1: + width = split[0] + height = split[1] + print(width) + print(height) + elif tag.as_vec()[1] == "model": + model = tag.as_vec()[2] + + prompt = prompt.replace(";", ",") + request_form['data'] = '[{"id":"input_prompt","type":"input","src":"request:text","data":"' + prompt + '","active":"True"},{"id":"negative_prompt","type":"input","src":"request:text","data":"' + negative_prompt + '","active":"True"},{"id":"output_image","type":"output","src":"request:image","active":"True"}]' + request_form["optStr"] = ('model=' + model + ';ratio=' + str(ratio_width) + '-' + str(ratio_height) + ';size=' + + str(width) + '-' + str(height) + ';strength=' + str(strength) + ';guidance_scale=' + + str(guidance_scale) + ';lora=' + lora + ';lora_weight=' + lora_weight) + + return request_form + + def process(self, request_form): + try: + # Call the process route of NOVA-Server with our request form. + success = send_request_to_nova_server(request_form, os.environ["NOVA_SERVER"]) + print(success) + + pool = ThreadPool(processes=1) + thread = pool.apply_async(check_nova_server_status, (request_form['jobID'], os.environ["NOVA_SERVER"])) + print("Wait for results of NOVA-Server...") + result = thread.get() + return str(result) + + except Exception as e: + raise Exception(e) diff --git a/tasks/textextractionPDF.py b/tasks/textextractionpdf.py similarity index 72% rename from tasks/textextractionPDF.py rename to tasks/textextractionpdf.py index 05ac8ce..26f8c45 100644 --- a/tasks/textextractionPDF.py +++ b/tasks/textextractionpdf.py @@ -2,12 +2,16 @@ import os import re from interfaces.dvmtaskinterface import DVMTaskInterface -from utils import env from utils.definitions import EventDefinitions from utils.nip89_utils import NIP89Announcement -from utils.nostr_utils import get_event_by_id, get_referenced_event_by_id +from utils.nostr_utils import get_event_by_id +""" +This File contains a Module to extract Text from a PDF file locally on the DVM Machine +Accepted Inputs: Url to pdf file, Event containing an URL to a PDF file +Outputs: Text containing the extracted contents of the PDF file +""" class TextExtractionPDF(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_EXTRACT_TEXT TASK: str = "pdf-to-text" @@ -17,16 +21,16 @@ class TextExtractionPDF(DVMTaskInterface): self.NAME = name self.PK = pk - def NIP89_announcement(self): + def NIP89_announcement(self, d_tag, content): nip89 = NIP89Announcement() nip89.kind = self.KIND nip89.pk = self.PK - nip89.dtag = os.getenv(env.TASK_TEXTEXTRACTION_NIP89_DTAG) - nip89.content = "{\"name\":\"" + self.NAME + "\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I extract Text from pdf documents\",\"nip90Params\":{}}" + nip89.dtag = d_tag + nip89.content = content return nip89 def is_input_supported(self, input_type, input_content): - if input_type != "url": + if input_type != "url" and input_type != "event": return False return True @@ -45,23 +49,21 @@ class TextExtractionPDF(DVMTaskInterface): if input_type == "url": url = input_content + # if event contains url to pdf, we checked for a pdf link before elif input_type == "event": evt = get_event_by_id(input_content, config=dvm_config) url = re.search("(?Phttps?://[^\s]+)", evt.content()).group("url") - elif input_type == "job": - evt = get_referenced_event_by_id(input_content, [EventDefinitions.KIND_NIP90_RESULT_GENERATE_IMAGE], - client, config=dvm_config) - - url = re.search("(?Phttps?://[^\s]+)", evt.content()).group("url") request_form["optStr"] = 'url=' + url return request_form def process(self, request_form): - options = DVMTaskInterface.setOptions(request_form) from pypdf import PdfReader from pathlib import Path import requests + + options = DVMTaskInterface.setOptions(request_form) + try: file_path = Path('temp.pdf') response = requests.get(options["url"]) @@ -76,4 +78,4 @@ class TextExtractionPDF(DVMTaskInterface): os.remove('temp.pdf') return text except Exception as e: - raise Exception(e) \ No newline at end of file + raise Exception(e) diff --git a/tasks/translation.py b/tasks/translation.py index 01e99ff..362aee8 100644 --- a/tasks/translation.py +++ b/tasks/translation.py @@ -1,12 +1,19 @@ import os from interfaces.dvmtaskinterface import DVMTaskInterface -from utils import env from utils.definitions import EventDefinitions from utils.nip89_utils import NIP89Announcement from utils.nostr_utils import get_referenced_event_by_id, get_event_by_id +""" +This File contains a Module to call Google Translate Services locally on the DVM Machine + +Accepted Inputs: Text, Events, Jobs (Text Extraction, Summary, Translation) +Outputs: Text containing the Translation in the desired language. +""" + + class Translation(DVMTaskInterface): KIND: int = EventDefinitions.KIND_NIP90_TRANSLATE_TEXT TASK: str = "translation" @@ -16,12 +23,12 @@ class Translation(DVMTaskInterface): self.NAME = name self.PK = pk - def NIP89_announcement(self): + def NIP89_announcement(self, d_tag, content): nip89 = NIP89Announcement() nip89.kind = self.KIND nip89.pk = self.PK - nip89.dtag = os.getenv(env.TASK_TRANSLATION_NIP89_DTAG) - nip89.content = "{\"name\":\"" + self.NAME + "\",\"image\":\"https://image.nostr.build/c33ca6fc4cc038ca4adb46fdfdfda34951656f87ee364ef59095bae1495ce669.jpg\",\"about\":\"I translate text from given text/event/job, currently using Google Translation Services into language defined in param. \",\"nip90Params\":{\"language\":{\"required\":true,\"values\":[\"af\",\"am\",\"ar\",\"az\",\"be\",\"bg\",\"bn\",\"bs\",\"ca\",\"ceb\",\"co\",\"cs\",\"cy\",\"da\",\"de\",\"el\",\"eo\",\"es\",\"et\",\"eu\",\"fa\",\"fi\",\"fr\",\"fy\",\"ga\",\"gd\",\"gl\",\"gu\",\"ha\",\"haw\",\"hi\",\"hmn\",\"hr\",\"ht\",\"hu\",\"hy\",\"id\",\"ig\",\"is\",\"it\",\"he\",\"ja\",\"jv\",\"ka\",\"kk\",\"km\",\"kn\",\"ko\",\"ku\",\"ky\",\"la\",\"lb\",\"lo\",\"lt\",\"lv\",\"mg\",\"mi\",\"mk\",\"ml\",\"mn\",\"mr\",\"ms\",\"mt\",\"my\",\"ne\",\"nl\",\"no\",\"ny\",\"or\",\"pa\",\"pl\",\"ps\",\"pt\",\"ro\",\"ru\",\"sd\",\"si\",\"sk\",\"sl\",\"sm\",\"sn\",\"so\",\"sq\",\"sr\",\"st\",\"su\",\"sv\",\"sw\",\"ta\",\"te\",\"tg\",\"th\",\"tl\",\"tr\",\"ug\",\"uk\",\"ur\",\"uz\",\"vi\",\"xh\",\"yi\",\"yo\",\"zh\",\"zu\"]}}}" + nip89.dtag = d_tag + nip89.content = content return nip89 def is_input_supported(self, input_type, input_content): @@ -65,7 +72,8 @@ class Translation(DVMTaskInterface): if tag.as_vec()[0] == 'i': evt = get_referenced_event_by_id(tag.as_vec()[1], [EventDefinitions.KIND_NIP90_RESULT_EXTRACT_TEXT, - EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT], + EventDefinitions.KIND_NIP90_RESULT_SUMMARIZE_TEXT, + EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT], client, config=dvm_config) text = evt.content() @@ -77,8 +85,9 @@ class Translation(DVMTaskInterface): return request_form def process(self, request_form): - options = DVMTaskInterface.setOptions(request_form) from translatepy.translators.google import GoogleTranslate + + options = DVMTaskInterface.setOptions(request_form) gtranslate = GoogleTranslate() length = len(options["text"]) diff --git a/test_client.py b/test_client.py index 4e7a4bf..29f3d97 100644 --- a/test_client.py +++ b/test_client.py @@ -1,4 +1,3 @@ - import os import time import datetime as datetime @@ -12,7 +11,9 @@ from utils.nostr_utils import send_event from utils.definitions import EventDefinitions, RELAY_LIST import utils.env as env -#TODO HINT: Only use this path with a preiously whitelisted privkey, as zapping events is not implemented in the lib/code + + +# TODO HINT: Best use this path with a previously whitelisted privkey, as zapping events is not implemented in the lib/code def nostr_client_test_translation(input, kind, lang, sats, satsmax): keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY)) if kind == "text": @@ -22,19 +23,46 @@ def nostr_client_test_translation(input, kind, lang, sats, satsmax): paramTag1 = Tag.parse(["param", "language", lang]) bidTag = Tag.parse(['bid', str(sats * 1000), str(satsmax * 1000)]) - relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", "wss://nostr-pub.wellorder.net"]) + relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"]) alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to translate a given Input"]) - event = EventBuilder(EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, str("Translate the given input."), [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys) + event = EventBuilder(EventDefinitions.KIND_NIP90_TRANSLATE_TEXT, str("Translate the given input."), + [iTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys) relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", "wss://nostr-pub.wellorder.net"] - client = Client(keys) for relay in relay_list: client.add_relay(relay) client.connect() - send_event(event, client, keys) + send_event(event, client, keys) + return event.as_json() + + +def nostr_client_test_image(prompt): + keys = Keys.from_sk_str(os.getenv(env.NOSTR_TEST_CLIENT_PRIVATE_KEY)) + + iTag = Tag.parse(["i", prompt, "text"]) + outTag = Tag.parse(["output", "image/png;format=url"]) + paramTag1 = Tag.parse(["param", "size", "1024x1024"]) + tTag = Tag.parse(["t", "bitcoin"]) + + bidTag = Tag.parse(['bid', str(1000 * 1000), str(1000 * 1000)]) + relaysTag = Tag.parse(['relays', "wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"]) + alttag = Tag.parse(["alt", "This is a NIP90 DVM AI task to translate a given Input"]) + event = EventBuilder(EventDefinitions.KIND_NIP90_GENERATE_IMAGE, str("Generate an Image."), + [iTag, outTag, tTag, paramTag1, bidTag, relaysTag, alttag]).to_event(keys) + + relay_list = ["wss://relay.damus.io", "wss://blastr.f7z.xyz", "wss://relayable.org", + "wss://nostr-pub.wellorder.net"] + + client = Client(keys) + for relay in relay_list: + client.add_relay(relay) + client.connect() + send_event(event, client, keys) return event.as_json() def nostr_client(): @@ -48,17 +76,19 @@ def nostr_client(): client.connect() dm_zap_filter = Filter().pubkey(pk).kinds([EventDefinitions.KIND_DM, - EventDefinitions.KIND_ZAP]).since(Timestamp.now()) # events to us specific + EventDefinitions.KIND_ZAP]).since( + Timestamp.now()) # events to us specific dvm_filter = (Filter().kinds([EventDefinitions.KIND_NIP90_RESULT_TRANSLATE_TEXT, - EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events + EventDefinitions.KIND_FEEDBACK]).since(Timestamp.now())) # public events client.subscribe([dm_zap_filter, dvm_filter]) + # nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) + #nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "es", 20, + # 20) - #nostr_client_test_translation("This is the result of the DVM in spanish", "text", "es", 20, 20) - nostr_client_test_translation("44a0a8b395ade39d46b9d20038b3f0c8a11168e67c442e3ece95e4a1703e2beb", "event", "fr", 20, 20) + # nostr_client_test_translation("9c5d6d054e1b7a34a6a4b26ac59469c96e77f7cba003a30456fa6a57974ea86d", "event", "zh", 20, 20) - - #nostr_client_test_image(sats=50, satsmax=10) + nostr_client_test_image("a beautiful purple ostrich watching the sunset") class NotificationHandler(HandleNotification): def handle(self, relay_url, event): print(f"Received new event from {relay_url}: {event.as_json()}") @@ -76,16 +106,14 @@ def nostr_client(): print("[Nostr Client]: " + f"Received new zap:") print(event.as_json()) - def handle_msg(self, relay_url, msg): - None + return client.handle_notifications(NotificationHandler()) while True: time.sleep(5.0) - if __name__ == '__main__': env_path = Path('.env') @@ -95,6 +123,5 @@ if __name__ == '__main__': else: raise FileNotFoundError(f'.env file not found at {env_path} ') - nostr_dvm_thread = Thread(target=nostr_client()) nostr_dvm_thread.start() diff --git a/utils/backend_utils.py b/utils/backend_utils.py index e0a0132..adcc9e1 100644 --- a/utils/backend_utils.py +++ b/utils/backend_utils.py @@ -1,10 +1,10 @@ import requests - -from tasks.textextractionPDF import TextExtractionPDF from utils.definitions import EventDefinitions from utils.nostr_utils import get_event_by_id +from tasks.textextractionpdf import TextExtractionPDF from tasks.translation import Translation +from tasks.imagegenerationsdxl import ImageGenerationSDXL def get_task(event, client, dvmconfig): @@ -35,9 +35,9 @@ def get_task(event, client, dvmconfig): evt = get_event_by_id(tag.as_vec()[1], config=dvmconfig) if evt is not None: if evt.kind() == 1063: - for tag in evt.tags(): - if tag.as_vec()[0] == 'url': - file_type = check_url_is_readable(tag.as_vec()[1]) + for tg in evt.tags(): + if tg.as_vec()[0] == 'url': + file_type = check_url_is_readable(tg.as_vec()[1]) if file_type == "pdf": return "pdf-to-text" else: @@ -45,9 +45,10 @@ def get_task(event, client, dvmconfig): else: return "unknown type" - elif event.kind() == EventDefinitions.KIND_NIP90_TRANSLATE_TEXT: return Translation.TASK + elif event.kind() == EventDefinitions.KIND_NIP90_GENERATE_IMAGE: + return ImageGenerationSDXL.TASK else: return "unknown type" @@ -121,7 +122,6 @@ def check_url_is_readable(url): def get_amount_per_task(task, dvm_config, duration=1): - print(dvm_config.SUPPORTED_TASKS) for dvm in dvm_config.SUPPORTED_TASKS: if dvm.TASK == task: amount = dvm.COST * duration diff --git a/utils/env.py b/utils/env.py index 1fef1d1..3d0bd1a 100644 --- a/utils/env.py +++ b/utils/env.py @@ -8,5 +8,6 @@ LNBITS_HOST = "LNBITS_HOST" TASK_TRANSLATION_NIP89_DTAG = "TASK_TRANSLATION_NIP89_DTAG" TASK_TEXTEXTRACTION_NIP89_DTAG = "TASK_TEXTEXTRACTION_NIP89_DTAG" +TASK_IMAGEGENERATION_NIP89_DTAG = "TASK_IMAGEGENERATION_NIP89_DTAG" diff --git a/utils/output_utils.py b/utils/output_utils.py index 06536f6..6f8c0f3 100644 --- a/utils/output_utils.py +++ b/utils/output_utils.py @@ -1,10 +1,15 @@ import json import datetime as datetime +import os from types import NoneType +import requests +from pyupload.uploader import CatboxUploader import pandas - +''' +Post process results to either given output format or a Nostr readable plain text. +''' def post_process_result(anno, original_event): print("post-processing...") if isinstance(anno, pandas.DataFrame): # if input is an anno we parse it to required output format @@ -84,7 +89,52 @@ def post_process_result(anno, original_event): return result +''' +Convenience function to replace words like Noster with Nostr +''' def replace_broken_words(text): result = (text.replace("Noster", "Nostr").replace("Nostra", "Nostr").replace("no stir", "Nostr"). replace("Nostro", "Nostr").replace("Impub", "npub").replace("sets", "Sats")) return result + + +''' +Function to upload to Nostr.build and if it fails to Nostrfiles.dev +Larger files than these hosters allow and fallback is catbox currently. +Will probably need to switch to another system in the future. +''' +def uploadMediaToHoster(filepath): + print("Uploading image: " + filepath) + try: + files = {'file': open(filepath, 'rb')} + file_stats = os.stat(filepath) + sizeinmb = file_stats.st_size / (1024*1024) + print("Filesize of Uploaded media: " + str(sizeinmb) + " Mb.") + if sizeinmb > 25: + uploader = CatboxUploader(filepath) + result = uploader.execute() + return result + else: + url = 'https://nostr.build/api/v2/upload/files' + response = requests.post(url, files=files) + json_object = json.loads(response.text) + result = json_object["data"][0]["url"] + return result + except: + try: + file = {'file': open(filepath, 'rb')} + url = 'https://nostrfiles.dev/upload_image' + response = requests.post(url, files=file) + json_object = json.loads(response.text) + print(json_object["url"]) + return json_object["url"] + # fallback filehoster + except: + + try: + uploader = CatboxUploader(filepath) + result = uploader.execute() + print(result) + return result + except: + return "Upload not possible, all hosters didn't work"