feat: stable

src/jina_cloud.py

@@ -1,16 +1,59 @@
+import hashlib
+import json
 import os
 import subprocess
 import re
+from argparse import Namespace
+from pathlib import Path
 
 import hubble
+from hubble.executor.helper import upload_file, archive_package, get_request_header
 from jcloud.flow import CloudFlow
 from jina import Flow
 
 
 
 def push_executor(dir_path):
-    cmd = f'jina hub push {dir_path}/. --verbose --replay'
-    os.system(cmd)
+    dir_path = Path(dir_path)
+
+    md5_hash = hashlib.md5()
+    bytesio = archive_package(dir_path)
+    content = bytesio.getvalue()
+    md5_hash.update(content)
+    md5_digest = md5_hash.hexdigest()
+
+    form_data = {
+        'public': 'True',
+        'private': 'False',
+        'verbose': 'True',
+        'md5sum': md5_digest,
+    }
+    req_header = get_request_header()
+    resp = upload_file(
+        'https://api.hubble.jina.ai/v2/rpc/executor.push',
+        'filename',
+        content,
+        dict_data=form_data,
+        headers=req_header,
+        stream=False,
+        method='post',
+    )
+    json_lines_str = resp.content.decode('utf-8')
+    if 'exited on non-zero code' not in json_lines_str:
+        return ''
+    responses = []
+    for json_line in json_lines_str.splitlines():
+        if 'exit code:' in json_line:
+            break
+
+        d = json.loads(json_line)
+
+        if 'payload' in d and type(d['payload']) == str:
+            responses.append(d['payload'])
+        elif type(d) == str:
+            responses.append(d)
+    return '\n'.join(responses)
+
 
 def get_user_name():
     client = hubble.Client(max_retries=None, jsonify=True)
@@ -51,10 +94,10 @@ executors:
     with open(full_flow_path, 'w') as f:
         f.write(flow)
 
-    print('try local execution')
-    flow = Flow.load_config(full_flow_path)
-    with flow:
-        pass
+    # print('try local execution')
+    # flow = Flow.load_config(full_flow_path)
+    # with flow:
+    #     pass
     print('deploy flow on jcloud')
     return deploy_on_jcloud(flow_yaml=full_flow_path)
 
@@ -78,22 +121,24 @@ def update_client_line_in_file(file_path, host):
         file.write(replaced_content)
 
 
+def process_error_message(error_message):
+    lines = error_message.split('\n')
+    relevant_lines = []
+
+    pattern = re.compile(r"^#\d+ \[[ \d]+/[ \d]+\]")  # Pattern to match lines like "#11 [7/8]"
+    last_matching_line_index = None
+
+    for index, line in enumerate(lines):
+        if pattern.match(line):
+            last_matching_line_index = index
+
+    if last_matching_line_index is not None:
+        relevant_lines = lines[last_matching_line_index:]
+
+    return '\n'.join(relevant_lines[-25:])
+
 def build_docker(path):
-    def process_error_message(error_message):
-        lines = error_message.split('\n')
-        relevant_lines = []
 
-        pattern = re.compile(r"^#\d+ \[[ \d]+/[ \d]+\]")  # Pattern to match lines like "#11 [7/8]"
-        last_matching_line_index = None
-
-        for index, line in enumerate(lines):
-            if pattern.match(line):
-                last_matching_line_index = index
-
-        if last_matching_line_index is not None:
-            relevant_lines = lines[last_matching_line_index:]
-
-        return '\n'.join(relevant_lines)
 
     # The command to build the Docker image
     cmd = f"docker build -t micromagic {path}"

src/prompt_tasks.py

@@ -27,11 +27,12 @@ def _task(task, tag_name, file_name):
     )
 
 
-def executor_file_task(executor_name, executor_description, input_modality, input_doc_field,
+def executor_file_task(executor_name, executor_description, test_scenario, input_modality, input_doc_field,
                        output_modality, output_doc_field):
     return _task(f'''
 Write the executor called '{executor_name}'.
 It matches the following description: '{executor_description}'.
+It will be tested with the following scenario: '{test_scenario}'.
 It gets a DocumentArray as input where each document has the input modality '{input_modality}' and can be accessed via document.{input_doc_field}.
 It returns a DocumentArray as output where each document has the output modality '{output_modality}' that is stored in document.{output_doc_field}.
 Have in mind that d.uri is never a path to a local file. It is always a url.